Skip to content

Commit 5b5f8df

Browse files
zhenfeizhangzhenfeizhang
authored
fix mpi ci (#174)
* remove rust cache * try... * clean up * profiling * the old fashion * Update install.py * remove apt update * Update benchmarks.yml * enable sudo for CI
1 parent 8abc8e2 commit 5b5f8df

File tree

5 files changed

+69
-20
lines changed

5 files changed

+69
-20
lines changed

‎.github/workflows/benchmarks.yml

+7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
name: Run Benchmarks
22

3+
env:
4+
RUSTFLAGS: "-Dwarnings -C target-cpu=native"
5+
RUST_BACKTRACE: 1
6+
ACTIONS_RUNNER_DEBUG: true
7+
38
on:
49
push:
510
pull_request:
@@ -29,6 +34,8 @@ jobs:
2934
run: cargo install cargo-criterion
3035
- name: Install mpi
3136
run: python3 ./scripts/install.py
37+
- name: Install build dependencies
38+
run: sudo apt-get install -y build-essential
3239
- name: Setup
3340
run: cargo run --bin=dev-setup --release
3441
- name: Run benchmark

‎.github/workflows/build.yml

+18-3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ on:
66
env:
77
RUSTFLAGS: "-Dwarnings -C target-cpu=native"
88
RUST_BACKTRACE: 1
9+
ACTIONS_RUNNER_DEBUG: true
910

1011
jobs:
1112
build:
@@ -23,18 +24,32 @@ jobs:
2324
runs-on: ${{ matrix.os }}
2425
steps:
2526
- uses: actions/checkout@v4
27+
28+
- name: Setup Dependencies
29+
run: |
30+
if [ "${{ matrix.os }}" == "macos-latest" ]; then
31+
brew install gcc make
32+
else
33+
sudo apt-get install -y build-essential
34+
fi
35+
shell: bash
36+
2637
- uses: dtolnay/rust-toolchain@stable
38+
2739
- uses: Swatinem/rust-cache@v2
2840
with:
29-
# The prefix cache key, this can be changed to start a new cache manually.
30-
prefix-key: "mpi-v5.0.6" # update me if brew formula changes to a new version
41+
prefix-key: "mpi-v5.0.6"
42+
3143
- name: Install MPI
3244
run: python3 ./scripts/install.py
45+
3346
- name: Set RUSTFLAGS for AVX
3447
if: matrix.feature != ''
3548
run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV
49+
3650
- name: Prepare binary
3751
run: cargo build --release --bin expander-exec
52+
3853
- name: Upload release asset
3954
uses: actions/github-script@v6
4055
with:
@@ -46,4 +61,4 @@ jobs:
4661
release_id: ${{ github.event.release.id }},
4762
name: '${{ matrix.binary_name }}',
4863
data: await fs.readFile('target/release/expander-exec')
49-
});
64+
});

‎.github/workflows/ci.yml

+28-6
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on: [pull_request, push]
55
env:
66
RUSTFLAGS: "-Dwarnings -C target-cpu=native"
77
RUST_BACKTRACE: 1
8+
ACTIONS_RUNNER_DEBUG: true
89

910
jobs:
1011
lint:
@@ -15,6 +16,9 @@ jobs:
1516
- uses: dtolnay/rust-toolchain@stable
1617
with:
1718
components: rustfmt, clippy
19+
- name: Setup Dependencies
20+
run: |
21+
sudo apt-get install -y build-essential
1822
- run: python3 ./scripts/install.py
1923
- run: cargo fmt --all -- --check
2024
- run: cargo clippy --all
@@ -35,18 +39,30 @@ jobs:
3539
- uses: dtolnay/rust-toolchain@stable
3640
- uses: Swatinem/rust-cache@v2
3741
with:
38-
# The prefix cache key, this can be changed to start a new cache manually.
39-
prefix-key: "mpi-v5.0.6" # update me if brew formula changes to a new version
42+
prefix-key: "mpi-v5.0.6"
43+
- name: Setup Dependencies
44+
run: |
45+
if [ "${{ matrix.os }}" == "macos-latest" ]; then
46+
brew install gcc make
47+
else
48+
sudo apt-get install -y build-essential
49+
fi
4050
- name: Set RUSTFLAGS for AVX
4151
if: matrix.feature != ''
4252
run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV
43-
- name: Build and Test
53+
- name: Setup
4454
run: |
4555
python3 ./scripts/install.py
4656
mpiexec --version
57+
- name: download data
58+
run: |
4759
cargo run --bin=dev-setup --release
60+
- name: Unit Tests
61+
run: |
4862
cargo build --all-features --release
4963
cargo test --all-features --release --workspace
64+
- name: E2E Test
65+
run: |
5066
./scripts/test_recursion.py
5167
5268
gkr-e2e:
@@ -84,13 +100,19 @@ jobs:
84100
- uses: dtolnay/rust-toolchain@stable
85101
- uses: Swatinem/rust-cache@v2
86102
with:
87-
# The prefix cache key, this can be changed to start a new cache manually.
88-
prefix-key: "mpi-v5.0.5" # update me if brew formula changes to a new version
103+
prefix-key: "mpi-v5.0.6"
104+
- name: Setup Dependencies
105+
run: |
106+
if [ "${{ matrix.os }}" == "macos-latest" ]; then
107+
brew install gcc make
108+
else
109+
sudo apt-get install -y build-essential
110+
fi
89111
- name: Set RUSTFLAGS for AVX
90112
if: matrix.feature != ''
91113
run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV
92114
- name: Run benchmark
93115
run: |
94116
python3 ./scripts/install.py
95117
cargo run --bin=dev-setup --release
96-
cargo run --bin=gkr --release -- -t ${{ matrix.os == 'macos-latest' && 2 || 16 }} -f ${{ matrix.field }}
118+
cargo run --bin=gkr --release -- -t ${{ matrix.os == 'macos-latest' && 2 || 16 }} -f ${{ matrix.field }}

‎.github/workflows/nightly_e2e.yml

+14-9
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
name: E2E Tests
22

3+
env:
4+
RUSTFLAGS: "-Dwarnings -C target-cpu=native"
5+
RUST_BACKTRACE: 1
6+
ACTIONS_RUNNER_DEBUG: true
7+
38
on:
49
schedule:
510
# This will run the workflow every day at 2:00 AM UTC
@@ -18,20 +23,26 @@ jobs:
1823
runs-on: 7950x3d
1924

2025
steps:
21-
- uses: actions/checkout@v2
26+
- uses: actions/checkout@v4 # Updated to v4
27+
28+
- name: Setup Dependencies
29+
run: |
30+
sudo apt-get update
31+
sudo apt-get install -y build-essential
2232
2333
- name: Set up Rust
2434
uses: dtolnay/rust-toolchain@master
2535
with:
2636
toolchain: nightly-2024-09-01
2737
components: rustfmt, clippy
38+
2839
- uses: Swatinem/rust-cache@v2
2940
with:
3041
# The prefix cache key, this can be changed to start a new cache manually.
3142
prefix-key: "mpi-v5.0.6" # update me if brew formula changes to a new version
3243

3344
- name: Set up Go
34-
uses: actions/setup-go@v2
45+
uses: actions/setup-go@v5 # Updated to v5
3546
with:
3647
go-version: '^1.16'
3748

@@ -65,10 +76,4 @@ jobs:
6576
RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f gf2ext128 -t 16
6677
RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f m31ext3 -t 16
6778
RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f fr -t 16
68-
RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s poseidon -f m31ext3 -t 16
69-
70-
# # Run MPI tests
71-
# RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f gf2ext128
72-
# RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f m31ext3
73-
# RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f fr
74-
# RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s poseidon -f m31ext3
79+
RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s poseidon -f m31ext3 -t 16

‎scripts/install.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@
33

44
if __name__ == "__main__":
55
if platform == "darwin": # mac os
6-
subprocess.run(["brew", "install", "openmpi"])
6+
subprocess.run(["brew", "install", "gcc", "make", "openmpi"])
77
else:
8-
pass # Do nothing, assuming mpi has already been installed
8+
pass

0 commit comments

Comments
 (0)