Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Bootstrap cache download ignores PARALLELISM #11030

Closed
wants to merge 1 commit into from
Closed

chore: Bootstrap cache download ignores PARALLELISM #11030

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,5 @@ tmux-client-*.log
.supermavenignore

# parallel
joblog.txt
joblog.txt
joblog-*.txt
160 changes: 122 additions & 38 deletions noir-projects/noir-protocol-circuits/bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,73 +51,154 @@ mkdir -p $key_dir
# Export vars needed inside compile.
export tmp_dir key_dir ci3 ivc_regex rollup_honk_regex

function compile {
# Tries downloading a circuit artifact from the remote cache
# Input: circuit directory
# Output: circuit name if successful, circuit name-hash otherwise
function try_download_circuit {
set -euo pipefail
local dir=$1
local name=${dir//-/_}
local filename="$name.json"
local json_path="./target/$filename"
local program_hash hash bytecode_hash vk vk_fields
local program_hash hash
local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2"
# echo_stderr $program_hash_cmd
program_hash=$(dump_fail "$program_hash_cmd")
echo_stderr "Hash preimage: $NARGO_HASH-$program_hash"
hash=$(hash_str "$NARGO_HASH-$program_hash")
if ! cache_download circuit-$hash.tar.gz 1>&2; then
SECONDS=0
rm -f $json_path
# TODO: --skip-brillig-constraints-check added temporarily for blobs build time.
local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check"
echo_stderr "$compile_cmd"
dump_fail "$compile_cmd"
echo_stderr "Compilation complete for: $name (${SECONDS}s)"
cache_upload circuit-$hash.tar.gz $json_path &> /dev/null

if cache_download circuit-$hash.tar.gz 1>&2; then
echo "$name"
else
echo "$name-$hash"
fi
}

# Compiles a circuit and uploads it to the remote cache if needed
# Input: circuit name or circuit name-hash, will only compile if hash is provided
# Output: circuit name
function compile_circuit {
set -euo pipefail
local name hash
IFS="-" read -r name hash <<< "$1"
if [[ -z $hash ]]; then
echo_stderr "Circuit $name downloaded from cache"
echo $name
return
fi

local filename="$name.json"
local json_path="./target/$filename"
echo_stderr "Compiling circuit $name-$hash to $json_path"

SECONDS=0
rm -f $json_path
# TODO: --skip-brillig-constraints-check added temporarily for blobs build time.
local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check"
echo_stderr "$compile_cmd"
dump_fail "$compile_cmd"
echo_stderr "Compilation complete for: $name (${SECONDS}s)"
cache_upload circuit-$hash.tar.gz $json_path &> /dev/null

echo $name
}

# Tries downloading a verification key from the remote cache
# Input: circuit name
# Output: circuit name if successful, circuit name-hash otherwise
function try_download_vk {
local name=$1
local filename="$name.json"
local json_path="./target/$filename"

# No vks needed for simulated circuits.
if [[ "$name" == *"simulated"* ]]; then
echo "$name"
return
fi

echo "$name"
# Compute protocol from name, used for the hash
if echo "$name" | grep -qE "${ivc_regex}"; then
local proto="client_ivc"
elif echo "$name" | grep -qE "${rollup_honk_regex}"; then
local proto="ultra_rollup_honk"
else
local proto="ultra_honk"
fi

local bytecode_hash=$(jq -r '.bytecode' $json_path | sha256sum | tr -d ' -')
local hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto")
if cache_download vk-$hash.tar.gz 1>&2; then
echo "$name"
else
echo "$name-$hash"
fi
}

# Generates a verification key and uploads it to the remote cache if needed
# Input: circuit name or circuit name-hash, will only compile if hash is provided
# Output: circuit name
function generate_vk {
set -euo pipefail
local name hash
IFS="-" read -r name hash <<< "$1"
if [[ -z $hash ]]; then
echo_stderr "Verification key $name downloaded from cache"
echo $name
return
fi

# Get commands for writing vk and converting to fields based on name
if echo "$name" | grep -qE "${ivc_regex}"; then
local write_vk_cmd="write_vk_for_ivc"
local vk_as_fields_cmd="vk_as_fields_mega_honk"
elif echo "$name" | grep -qE "${rollup_honk_regex}"; then
local proto="ultra_rollup_honk"
local write_vk_cmd="write_vk_ultra_rollup_honk -h 2"
local vk_as_fields_cmd="vk_as_fields_ultra_rollup_honk"
else
local proto="ultra_honk"
local write_vk_cmd="write_vk_ultra_honk -h 1"
local vk_as_fields_cmd="vk_as_fields_ultra_honk"
fi
echo "$proto$"

# No vks needed for simulated circuits.
[[ "$name" == *"simulated"* ]] && return
local filename="$name.json"
local json_path="./target/$filename"
local key_path="$key_dir/$name.vk.data.json"
echo_stderr "Generating vk for function: $name..."

# Change this to add verification_key to original json, like contracts does.
# Will require changing TS code downstream.
bytecode_hash=$(jq -r '.bytecode' $json_path | sha256sum | tr -d ' -')
hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto")
if ! cache_download vk-$hash.tar.gz 1>&2; then
local key_path="$key_dir/$name.vk.data.json"
echo_stderr "Generating vk for function: $name..."
SECONDS=0
local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -b - -o - --recursive | xxd -p -c 0"
echo_stderr $vk_cmd
vk=$(dump_fail "$vk_cmd")
local vkf_cmd="echo '$vk' | xxd -r -p | $BB $vk_as_fields_cmd -k - -o -"
# echo_stderrr $vkf_cmd
vk_fields=$(dump_fail "$vkf_cmd")
jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path
echo_stderr "Key output at: $key_path (${SECONDS}s)"
cache_upload vk-$hash.tar.gz $key_path &> /dev/null
fi
local vk vk_fields
SECONDS=0
local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -b - -o - --recursive | xxd -p -c 0"
echo_stderr $vk_cmd
vk=$(dump_fail "$vk_cmd")
local vkf_cmd="echo '$vk' | xxd -r -p | $BB $vk_as_fields_cmd -k - -o -"
# echo_stderrr $vkf_cmd
vk_fields=$(dump_fail "$vkf_cmd")
jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path
echo_stderr "Key output at: $key_path (${SECONDS}s)"
cache_upload vk-$hash.tar.gz $key_path &> /dev/null

echo $name
}

# Compiles a single circuit
function compile {
set -eu
[ -f "package.json" ] && denoise "yarn && node ./scripts/generate_variants.js"
local dir=$1
generate_vk $(try_download_vk $(compile_circuit $(try_download_circuit $dir)))
}

# Runs the following steps for each circuit:
# 1. try_download_circuit
# 2. compile_circuit
# 3. try_download_vk
# 4. generate_vk
function build {
set +e
set -u

[ -f "package.json" ] && denoise "yarn && node ./scripts/generate_variants.js"
local parallel_cmd="parallel -t --line-buffer --halt now,fail=1"
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had to remove --tag since it polluted stdout. With --tag set, the input to the second command (compile_circuit) included the tag. However, by removing the tag we now don't get any feedback on what circuit are we running.

Any idea on how to make --tag only apply to stderr? Or should I arrange this in a different way? An easy solution is to manually include the circuit name in every echo_stderr we do here, since there are not too many. But I'd like to know what's the proper way of handling this.


grep -oP '(?<=crates/)[^"]+' Nargo.toml | \
while read -r dir; do
Expand All @@ -126,9 +207,12 @@ function build {
echo "$(basename $dir)"
fi
done | \
parallel -j$PARALLELISM --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {}
$parallel_cmd -j32 --joblog joblog-1.txt try_download_circuit {} | \
$parallel_cmd -j$PARALLELISM --joblog joblog-2.txt compile_circuit {} | \
$parallel_cmd -j32 --joblog joblog-3.txt try_download_vk {} | \
$parallel_cmd -j$PARALLELISM --joblog joblog-4.txt generate_vk {}
code=$?
cat joblog.txt
cat joblog-*.txt
return $code
}

Expand All @@ -142,7 +226,7 @@ function test {
cache_upload_flag $name-tests-$CIRCUITS_HASH
}

export -f compile test build
export -f try_download_circuit compile_circuit try_download_vk generate_vk test build

case "$CMD" in
"clean")
Expand Down
Loading