Merge branch 'master' into jf/numeric-type

.github/scripts/merge-bench-reports.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -eu
+
+echo "Merging reports"
+
+REPORT_NAME=$1
+NAME_PLURAL=""$REPORT_NAME"s"
+
+combined_reports="[]"
+
+# Iterate over each report and merge them
+for report in ./reports/*; do
+  # The report is saved under ./$REPORT_NAME_{ matrix_report }/$REPORT_NAME_{ matrix_report }.json
+  FILE_PATH=$(echo $(ls $report)) 
+
+  # Extract the $NAME_PLURAL array from each report and merge it
+  combined_reports=$(jq '[."'"$NAME_PLURAL"'"[]] + '"$combined_reports" <<< "$(cat "$report/$FILE_PATH")")
+done
+
+combined_reports=$(jq '[."'$NAME_PLURAL'"[]] + '"$combined_reports" <<< "$(cat ./$REPORT_NAME.json)")
+
+# Wrap the merged memory reports into a new object as to keep the $NAME_PLURAL key
+final_report="{\"$NAME_PLURAL\": $combined_reports}"
+
+echo "$final_report" > $REPORT_NAME.json
+
+cat $REPORT_NAME.json

.github/workflows/reports.yml

@@ -38,7 +38,6 @@ jobs:
           path: ./dist/*
           retention-days: 3
 
-
   compare_gates_reports:
     name: Circuit sizes
     needs: [build-nargo]
@@ -218,21 +217,13 @@ jobs:
           ./memory_report.sh
           mv memory_report.json ../memory_report.json
 
-      - name: Parse memory report
-        id: memory_report
-        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
-        with:
-          report: memory_report.json
-          header: |
-            # Memory Report
-          memory_report: true
-
-      - name: Add memory report to sticky comment
-        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
-        uses: marocchino/sticky-pull-request-comment@v2
+      - name: Upload memory report
+        uses: actions/upload-artifact@v4
         with:
-          header: memory
-          message: ${{ steps.memory_report.outputs.markdown }}
+          name: in_progress_memory_report
+          path: memory_report.json
+          retention-days: 3
+          overwrite: true
 
   generate_compilation_report:
     name: Compilation time
@@ -262,8 +253,116 @@ jobs:
         working-directory: ./test_programs
         run: |
           ./compilation_report.sh
+          cat compilation_report.json
           mv compilation_report.json ../compilation_report.json
 
+      - name: Upload compilation report
+        uses: actions/upload-artifact@v4
+        with:
+          name: in_progress_compilation_report
+          path: compilation_report.json
+          retention-days: 3
+          overwrite: true
+
+  external_repo_compilation_report:
+    needs: [build-nargo]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
+          # TODO: Bring these back once they no longer time out
+          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
+          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }
+
+    name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
+    steps:
+      - name: Download nargo binary
+        uses: actions/download-artifact@v4
+        with:
+          name: nargo
+          path: ./nargo
+
+      - name: Set nargo on PATH
+        run: |
+          nargo_binary="${{ github.workspace }}/nargo/nargo"
+          chmod +x $nargo_binary
+          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
+          export PATH="$PATH:$(dirname $nargo_binary)"
+          nargo -V
+
+      - uses: actions/checkout@v4
+        with:
+          path: scripts
+          sparse-checkout: |
+            test_programs/compilation_report.sh
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.project.repo }}
+          path: test-repo
+          ref: ${{ matrix.project.ref }}
+
+      - name: Generate compilation report
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
+          chmod +x ./compilation_report.sh
+          ./compilation_report.sh 1
+
+      - name: Move compilation report
+        id: report
+        shell: bash
+        run: |
+          PACKAGE_NAME=${{ matrix.project.path }}
+          PACKAGE_NAME=$(basename $PACKAGE_NAME)
+          mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json
+          echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Upload compilation report
+        uses: actions/upload-artifact@v4
+        with:
+          name: compilation_report_${{ steps.report.outputs.compilation_report_name }}
+          path: compilation_report_${{ steps.report.outputs.compilation_report_name }}.json
+          retention-days: 3
+          overwrite: true
+
+  upload_compilation_report:
+    name: Upload compilation report 
+    needs: [generate_compilation_report, external_repo_compilation_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_compilation_report` fails
+    if: always() 
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download initial compilation report
+        uses: actions/download-artifact@v4
+        with:
+          name: in_progress_compilation_report
+
+      - name: Download matrix compilation reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: compilation_report_*
+          path: ./reports
+
+      - name: Merge compilation reports using jq
+        run: |
+          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
+          ./merge-bench-reports.sh compilation_report
+
       - name: Parse compilation report
         id: compilation_report
         uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
@@ -279,3 +378,116 @@ jobs:
         with:
           header: compilation
           message: ${{ steps.compilation_report.outputs.markdown }}
+
+  external_repo_memory_report:
+    needs: [build-nargo]
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes.
+          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
+
+    name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
+    steps:
+      - name: Download nargo binary
+        uses: actions/download-artifact@v4
+        with:
+          name: nargo
+          path: ./nargo
+
+      - name: Set nargo on PATH
+        run: |
+          nargo_binary="${{ github.workspace }}/nargo/nargo"
+          chmod +x $nargo_binary
+          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
+          export PATH="$PATH:$(dirname $nargo_binary)"
+          nargo -V
+
+      - uses: actions/checkout@v4
+        with:
+          path: scripts
+          sparse-checkout: |
+            test_programs/memory_report.sh
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.project.repo }}
+          path: test-repo
+          ref: ${{ matrix.project.ref }}
+
+      - name: Generate compilation report
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
+          chmod +x ./memory_report.sh
+          ./memory_report.sh 1
+
+      - name: Move compilation report
+        id: report
+        shell: bash
+        run: |
+          PACKAGE_NAME=${{ matrix.project.path }}
+          PACKAGE_NAME=$(basename $PACKAGE_NAME)
+          mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
+          echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Upload memory report
+        uses: actions/upload-artifact@v4
+        with:
+          name: memory_report_${{ steps.report.outputs.memory_report_name }}
+          path: memory_report_${{ steps.report.outputs.memory_report_name }}.json
+          retention-days: 3
+          overwrite: true
+
+  upload_memory_report:
+    name: Upload memory report 
+    needs: [generate_memory_report, external_repo_memory_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails
+    if: always() 
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download initial memory report
+        uses: actions/download-artifact@v4
+        with:
+          name: in_progress_memory_report
+
+      - name: Download matrix memory reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: memory_report_*
+          path: ./reports
+
+      - name: Merge memory reports using jq
+        run: |
+          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
+          ./merge-bench-reports.sh memory_report
+
+      - name: Parse memory report
+        id: memory_report
+        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
+        with:
+          report: memory_report.json
+          header: |
+            # Memory Report
+          memory_report: true
+
+      - name: Add memory report to sticky comment
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: memory
+          message: ${{ steps.memory_report.outputs.markdown }}

test_programs/compilation_report.sh

@@ -3,12 +3,18 @@ set -e
 
 current_dir=$(pwd)
 base_path="$current_dir/execution_success"
-test_dirs=$(ls $base_path)
 
 # Tests to be profiled for compilation report
 tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression")
+
 echo "{\"compilation_reports\": [ " > $current_dir/compilation_report.json
 
+# If there is an argument that means we want to generate a report for only the current directory
+if [ "$#" -ne 0 ]; then
+  base_path="$current_dir"
+  tests_to_profile=(".")
+fi
+
 ITER="1"
 NUM_ARTIFACTS=${#tests_to_profile[@]}
 
@@ -23,9 +29,14 @@ for dir in ${tests_to_profile[@]}; do
 
     cd $base_path/$dir
 
-    COMPILE_TIME=$((time nargo compile --force) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s')
-    echo -e " {\n    \"artifact_name\":\"$dir\",\n    \"time\":\"$COMPILE_TIME\"\n" >> $current_dir/compilation_report.json
+    PACKAGE_NAME=$dir
+    if [ "$#" -ne 0 ]; then
+      PACKAGE_NAME=$(basename $current_dir)
+    fi
 
+    COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s')
+    echo -e " {\n    \"artifact_name\":\"$PACKAGE_NAME\",\n    \"time\":\"$COMPILE_TIME\"" >> $current_dir/compilation_report.json
+    
     if (($ITER == $NUM_ARTIFACTS)); then
         echo "}" >> $current_dir/compilation_report.json
     else 
@@ -36,4 +47,3 @@ for dir in ${tests_to_profile[@]}; do
 done
 
 echo "]}" >> $current_dir/compilation_report.json
-

test_programs/memory_report.sh

@@ -5,29 +5,36 @@ sudo apt-get install heaptrack
 
 NARGO="nargo"
 
-
 # Tests to be profiled for memory report
 tests_to_profile=("keccak256" "workspace" "regression_4709" "ram_blowup_regression")
 
 current_dir=$(pwd)
-execution_success_path="$current_dir/execution_success"
-test_dirs=$(ls $execution_success_path)
+base_path="$current_dir/execution_success"
+
+# If there is an argument that means we want to generate a report for only the current directory
+if [ "$#" -ne 0 ]; then
+  base_path="$current_dir"
+  tests_to_profile=(".")
+fi
 
 FIRST="1"
 
 echo "{\"memory_reports\": [ " > memory_report.json
 
-
 for test_name in ${tests_to_profile[@]}; do    
-        full_path=$execution_success_path"/"$test_name
-        cd $full_path
+        cd $base_path/$test_name
 
         if [ $FIRST = "1" ]
         then
             FIRST="0"
         else
             echo " ," >> $current_dir"/memory_report.json"
         fi
+
+        if [ "$#" -ne 0 ]; then
+            test_name=$(basename $current_dir)
+        fi
+
         heaptrack --output $current_dir/$test_name"_heap" $NARGO compile --force
         if test -f $current_dir/$test_name"_heap.gz"; 
         then