From 72fba0c8d86d16f70c0d044cf77fa6eb1afea525 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Tue, 8 Aug 2023 12:09:40 -0700 Subject: [PATCH 01/52] Update builder_bazel_slsa3.yml Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- .github/workflows/builder_bazel_slsa3.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/builder_bazel_slsa3.yml b/.github/workflows/builder_bazel_slsa3.yml index 129a13f922..6e40b001d8 100644 --- a/.github/workflows/builder_bazel_slsa3.yml +++ b/.github/workflows/builder_bazel_slsa3.yml @@ -35,6 +35,11 @@ on: required: false type: string default: "" + docker-image: + description: "Docker Image for build environment to run on" + required: false + type: string + default: "" needs-runfiles: description: > A boolean input that if true will package the artifact's runfiles along with the artifact. @@ -76,6 +81,19 @@ on: When run on other triggers, attestations are signed and have an "intoto.sigstore" extension. value: ${{ jobs.slsa-run.outputs.attestations-download-name }} + provenance-download-sha256: + description: > + The sha256 digest of the attestations. + + Users should verify the download against this digest to prevent tampering. + value: ${{ jobs.slsa-run.outputs.attestations-download-sha256 }} + + binaries-download-name: + description: > + The name of the folder containing the built artifacts. There is a random hash at the + beginning of it in form -binaries to avoid collisions. + value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).binaries-download-name }} + jobs: slsa-setup: permissions: From a70c7502e9f4b9ee36250fc315825993abe3ec1a Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Tue, 8 Aug 2023 12:13:23 -0700 Subject: [PATCH 02/52] Create rebuilder.sh Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/rebuilder.sh | 490 +++++++++++++++++++++++++++ 1 file changed, 490 insertions(+) create mode 100644 internal/builders/bazel/rebuilder.sh diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh new file mode 100644 index 0000000000..4cca841caf --- /dev/null +++ b/internal/builders/bazel/rebuilder.sh @@ -0,0 +1,490 @@ +#!/bin/bash +# +# Copyright 2023 SLSA Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: -u not set to check for empty variables from parse arguments function. +set -eo pipefail + +# Disabled to stop triggering warnings about color env vars. + +# This directory is where the rebuilt artifacts will be stored. It is made upon +# running the rebuilder. The long name is to avoid potential collisions. +rebuilt_artifacts_dir="rebuilt_artifacts_0ffe97cd2693d6608f5a787151950ed8" +mkdir $rebuilt_artifacts_dir + +################################################ +# # +# Color Code Env Vars # +# # +################################################ + +RESET="\033[0m" +BOLD="\033[1m" +RED="\033[1;31m" +LIGHT_RED="\033[0;31m" +GREEN="\033[1;32m" +YELLOW="\033[1;33m" +BLUE="\033[1;34m" +CYAN="\033[1;36m" +MAGENTA="\033[1;35m" +PURPLE="\033[1;35m" +BOLD_RED_BG="\033[1;41m" +UNDERLINE="\033[4m" + +################################################ +# # +# To Output Styled Progress # +# # +################################################ + +TYPE_SPEED=0.02 +function type_writer { + text="$1" + + for (( i=0; i<${#text}; i++ )); do + echo -n "${text:$i:1}" + sleep $TYPE_SPEED + done + echo "" +} + +################################################ +# # +# Process Arguments # +# # +################################################ + +# For later - after rebuild - to cd into binaries folder to obtain artifacts. +# The long name is to avoid potential collisions. +binaries_dir="bazel_builder_binaries_to_upload_to_gh_7bc972367cb286b7f36ab4457f06e369" + +# Boolean that decides whether to use the slsa-verifier in addition to rebuild. +verify=0 + +# Boolean to trigger verbose version of Rebuilder. +verbose=0 + +# Boolean to trigger cleanup upon completion or failure. +cleanup=0 + +# Outputs the usage of the Rebuilder script for the two modes: +# 1) Verify and Rebuild +# 2) Rebuild only +function usage() { + if [[ $verify ]] + then + # Disabled to stop triggering warnings about color env vars. + # shellcheck disable=SC2059 + printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}\n" + printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}\n" "$0" + printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}\n" + printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} \n" "$0" + else + # shellcheck disable=SC2059 + printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}\n" + printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} \n" "$0" + printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}\n" + printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}\n" "$0" + fi +} + +# Processes an argument for the script. Returns 0 if the given argument +# was recognized as an argument for this script, and 1 if it was not. +function process_argument() { + case "$1" in + --artifact_path=*) artifact_path="${1#--artifact_path=}" ;; + --prov_path=*) prov_path="${1#--prov_path=}" ;; + --source_uri=*) source_uri="${1#--source_uri=}" ;; + --builder_id=*) builder_id="${1#--builder_id=}" ;; + --docker_image=*) docker_image="${1#--docker_image=}" ;; + --verify) verify=1 ;; + --verbose) verbose=1 ;; + --cleanup) cleanup=1 ;; + + *) + return 1 ;; + esac + return 0 +} + +# This function is will clean up built directories off after error. +function cleanup() { + # If the cleanup flag is specified, remove created directories. + if [[ $cleanup -eq 1 ]] + then + type_writer "๐Ÿงน---> Cleaning up $rebuilt_artifacts_dir..." + rm -rf $rebuilt_artifacts_dir + + type_writer "๐Ÿงน---> Cleaning up $repo_name..." + sudo rm -rf "$repo_name" + + type_writer "๐Ÿงน---> Cleaning up slsa-verifier..." + sudo rm -rf slsa-verifier + fi +} + +# Parse arguments sequentially to check for unrecognized arguments +for ARG in "$@"; do + returnValue=$? + process_argument "$ARG" + if [[ ! ($returnValue) ]] + then + my_arg="$ARG" + printf "${RED}[ERROR] ${LIGHT_RED}%s is unrecognized${RESET}\n" "$my_arg" + usage + exit 1 + fi +done + +################################################ +# # +# Check Usage and Output Verbose # +# # +################################################ + +# Check if mandatory arguments for rebuild are not empty +if [ -z "$artifact_path" ]; then + printf "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --artifact_path, is missing or empty${RESET}\n" + usage + exit 1 +fi + +if [ -z "$prov_path" ]; then + printf "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --prov_path, is missing or empty${RESET}\n" + usage + exit 1 +fi + +if [ -z "$source_uri" ]; then + printf "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}\n" + usage + exit 1 +fi + +# Check if mandatory arguments for verification are not empty +if [[ $verify -eq 1 && ( -z "$source_uri" || -z "$builder_id" ) ]] +then + printf "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}\n" + usage + exit 1 +fi + +# Print received arguments (optional) +if [[ $verbose -eq 1 ]] +then + printf "${BLUE}โœ” Input Arguments Received:${RESET}\n" + printf "${CYAN}artifact_path: ${GREEN}%s${RESET}\n" "$artifact_path" + printf "${CYAN}prov_path: ${GREEN}%s${RESET}\n" "$prov_path" + printf "${CYAN}source_uri: ${GREEN}%s${RESET}\n" "$source_uri" + + if [ -n "$builder_id" ]; then + printf "${CYAN}builder_id: ${GREEN}%s${RESET}\n" "$builder_id" + fi + + if [ -n "$docker_image" ]; then + printf "${CYAN}docker_image: ${GREEN}%s${RESET}\n" "$docker_image" + fi + + printf "${CYAN}verify: ${GREEN}%s${RESET}\n" "$verify" + printf "${CYAN}verbose: ${GREEN}%s${RESET}\n" "$verbose" + printf "${CYAN}cleanup: ${GREEN}%s${RESET}\n" "$cleanup" + echo "" +fi + +################################################ +# # +# Use Verifier (if --verify) # +# # +################################################ + +if [[ $verify -eq 1 ]] +then + # Clone the slsa-verifier repository + if [ -d "slsa-verifier" ]; then + type_writer "๐Ÿ“---> The slsa-verifier repository is already cloned." + type_writer "โš ๏ธ---> To verify please remove the collision and try again" + exit 1 + else + printf "${CYAN}====================================================${RESET}\n" + type_writer "๐Ÿ“ฅ---> The slsa-verifier repository is not cloned. Cloning..." + git clone https://github.com/enteraga6/slsa-verifier + fi + + # Change directory to the slsa-verifier directory + cd slsa-verifier + + # Run SLSA Verifier on user inputs + # write if builder id then this if not include builder id then other command + # this is for once the non-compulsory feature gets merged. + go run ./cli/slsa-verifier/ verify-artifact ../"$artifact_path" --provenance-path ../"$prov_path" --source-uri "$source_uri" --builder-id "$builder_id" + + cd .. + printf "${CYAN}====================================================${RESET}\n" + echo "" +fi + +# Compute the original checksum of the artifact to compare with Rebuilt. +orig_checksum=$(sha256sum "$artifact_path" | awk '{ print $1 }') + +################################################ +# # +# Parse Provenance # +# # +################################################ + +# Associative Array to store the inputs to the GH workflow in key:value form +declare -A data + +# Extract the inputs and put them in data map where the key is the workflow input, +# and the value is the value that the user inputted. Pipe value to @text, to deal with booleans. +while IFS='=' read -r key value; do + data["$key"]="$value" +done < <(cat "$prov_path" | jq -r '.dsseEnvelope.payload' | base64 -d | jq -r '.predicate.buildDefinition.externalParameters.inputs | to_entries | .[] | .key + "=" + (.value | @text)') + +# Todo: Style Env Vars Later + +if [[ $verbose -eq 1 ]] +then + printf "${PURPLE}โœ” Arguments Parsed from Provenance:${RESET}\n" + for key in "${!data[@]}" + do + printf "${MAGENTA}$key: ${GREEN}%s${RESET}\n" "${data[$key]}" + done + echo "" +fi + +################################################ +# # +# Setup ENV Vars # +# # +################################################ + +# The name map will convert and export the key strings of inputs to +# match with the environment variables of the Bazel Builder build.sh +declare -A name_mapping +name_mapping["targets"]="TARGETS" +name_mapping["flags"]="FLAGS" +name_mapping["docker-image"]="DOCKER_IMAGE" + +# Note: These boolean inputs are now dealed with as strings +name_mapping["includes-java"]="INCLUDES_JAVA" +name_mapping["needs-runfiles"]="NEEDS_RUNFILES" + +# Export the inputs for later use +for key in "${!data[@]}"; do + # Check to see if the key is in name map before export as env var. + if [[ ${name_mapping[$key]+_} ]]; then + export "${name_mapping[$key]}"="${data[$key]}" + fi +done + +################################################ +# # +# Clone Repo to Rebuild # +# # +################################################ + +repo_name=$(basename "$source_uri") +# Clone the source_uri repository to begin rebuild process +if [ -d "$repo_name" ]; then + printf "${CYAN}====================================================${RESET}\n" + type_writer "๐Ÿ“---> Source repository appears already." + type_writer "โš ๏ธ---> To run rebuilder, fix collision by removing directory with name of $repo_name." + exit 1 +else + printf "${CYAN}====================================================${RESET}\n" + type_writer "๐Ÿ‘---> Cloning the source repository..." + echo "" + git clone https://"$source_uri" + printf "${CYAN}====================================================${RESET}\n" + echo "" +fi + +# Enter the Repo +cd "$repo_name" + +# Check to see if JAVA_HOME is set then empty to +# avoid triggering unbound variable error. +if [[ "${INCLUDES_JAVA}" == "true" ]] +then + if [[ ! -v JAVA_HOME || -z "${JAVA_HOME}" ]] + then + # if JAVA_HOME is empty, set to jdk bin path from $(which java) + if java_path=$(which java); then + JAVA_HOME="$(dirname "$(dirname "${java_path}")")" + export JAVA_HOME + # JAVA_HOME cannot be set automatically + else + echo "JAVA_HOME cannot be set automatically. Check PATH." + fi + else + echo "JAVA_HOME already set to ${JAVA_HOME}" + fi +fi + +################################################ +# # +# Rebuild the Artifacts # +# # +################################################ + +echo "" +printf "${CYAN}======================================================${RESET}\n" +printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} ๐Ÿ”จ Starting the Rebuild Process ๐Ÿ”จ ${RESET}${CYAN}|${RESET}\n" +printf "${CYAN}======================================================${RESET}\n" + +# Conditionals for docker images depend on if a Docker Image was use to build on Github. +# If a Docker Image was not used to build on Github, then build locally. This is done to +# ensure consistent build environment between both platforms. +if [[ -n $DOCKER_IMAGE ]] +then + cd - + sudo docker pull "$DOCKER_IMAGE" + echo "" + printf "${CYAN}====================================================${RESET}\n" + type_writer "๐Ÿ”จ---> Rebuilding with Docker Image Environment..." # Mount docker image on this directory as workdir to gain access to script env + printf "${CYAN}====================================================${RESET}\n" + echo "" + + sudo docker run --env repo_name="$repo_name" --env TARGETS="${TARGETS}" --env FLAGS="${FLAGS}" --env NEEDS_RUNFILES="${NEEDS_RUNFILES}" --env INCLUDES_JAVA="${INCLUDES_JAVA}" --rm -v "$PWD":/workdir -w /workdir "$DOCKER_IMAGE" /bin/sh -c "cd $repo_name && ./../build.sh" + echo "" + printf "${CYAN}=============================================${RESET}\n" + printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}\n" + printf "${CYAN}=============================================${RESET}\n" + echo "" +else + if [[ -n "$docker_image" ]] + then + # Warning message for the users if their artifact was not built with a Docker Image, but a Docker Image was provided at command. + printf "${RED}[Warning] ${LIGHT_RED}Docker Image, %s, provided, but artifact was not originally built on Docker Image${RESET}\n" "$docker_image" + else + echo "" # This is just for style. + fi + + # Run the build script locally without a docker image. + printf "${CYAN}=============================================${RESET}\n" + type_writer "๐Ÿ’ป---> Rebuilding with local environment..." + printf "${CYAN}=============================================${RESET}\n" + echo "" + + # shellcheck source=../build.sh + source ../build.sh + echo "" + printf "${CYAN}=============================================${RESET}\n" + printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}\n" + printf "${CYAN}=============================================${RESET}\n" + echo "" +fi + +# To avoid unbound variable after build script which sets -euo. +set +u + +# If Docker Image was used to build on Github, we need to cd into repo +# to access the binaries directory. +if [[ -n $DOCKER_IMAGE ]] +then + cd "$repo_name" +fi + +################################################ +# # +# Copy the Artifact # +# # +################################################ + +# Obtain the name of the artifact +if [[ $artifact_path == */* ]] +then + artifact_name=$(basename "$artifact_path") +else + artifact_name=$artifact_path +fi + +rebuilt_checksum="" +unset rebuilt_checksum # Makes sure it is empty before assigning. + +# IF there are runfiles, the directory structure will be different. +# The binaries folder contains different directories for the its artifacts and +# the artifacts runfiles. Obtain the rebuilt binaries and copy them to the +# path at root before cleaning up and deleting the repo. +if [[ "$artifact_name" == *"_deploy.jar"* ]] +then + # Uses _deploy.jar as a field seperator and grabs the field before it. + # Directory of Java artifacts is same as run script name. + run_script_name=$(echo "$artifact_name" | awk -F'_deploy.jar' '{print $1}') + cd $binaries_dir/ + rebuilt_checksum=$(sha256sum ./"$run_script_name"/"$artifact_name" | awk '{ print $1 }') + + # Copy the entire directory, including the run script. + cp -R ./"$run_script_name" ./../../"$rebuilt_artifacts_dir"/ +else + if [[ "${NEEDS_RUNFILES}" == "true" ]] + then + # For non-java targets with runfiles. + cd $binaries_dir/ + rebuilt_checksum=$(sha256sum ./"$artifact_name"/"$artifact_name" | awk '{ print $1 }') + + # Copy entire directory, including the runfiles. + cp -R ./"$artifact_name" ./../../"$rebuilt_artifacts_dir"/ + else + # For files withouts runfiles. + cd $binaries_dir + rebuilt_checksum=$(sha256sum "$artifact_name" | awk '{ print $1 }') + + cp "$artifact_name" ./../../$rebuilt_artifacts_dir/ + fi +fi + +################################################ +# # +# Check Build for Reproducibility # +# # +################################################ + +if [[ "$orig_checksum" == "$rebuilt_checksum" ]] +then + printf "${GREEN}Checksum is the ${BOLD}${UNDERLINE}same${RESET}${GREEN} for the original and rebuilt artifact!${RESET}\n" + printf "${GREEN}โœ… This build is ${BOLD}${UNDERLINE}reproducible!${RESET} โœ… \n" + echo "" + printf "${GREEN}%s${RESET} = Original Checksum${RESET}\n" "$orig_checksum" + printf "${GREEN}%s${RESET} = Rebuilt Checksum${RESET}\n" "$rebuilt_checksum" + echo "" +else + printf "${BOLD_RED_BG}Checksum is ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} the same for the original and rebuilt artifact!${RESET}\n" + printf "${BOLD_RED_BG} โš ๏ธ This build was ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} able to be reproduced! โš ๏ธ ${RESET}\n" + echo "" + printf "${RED}%s${RESET} = Original Checksum\n" "$orig_checksum" + printf "${RED}%s${RESET} = Rebuilt Checksum\n" "$rebuilt_checksum" + echo "" +fi + + +if [[ cleanup -eq 1 ]] +then + + # If there are runfiles or if the artifacts are Java, then each artifact + # has its own directory, so you need to exit out of it first. + if [[ "${NEEDS_RUNFILES}" == "true" || "${INCLUDES_JAVA}" == "true" ]] + then + cd .. + fi + + # Current position is bazel_builder_dir/$repo_name/$binaries_dir, + # and to clean up need to be in /bazel. + cd ../.. + + # Now cleanup of verifier and cloned $repo_name. + cleanup +fi From bb8b2eb9192ced26aeb3b80598d5a60ece18c980 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Tue, 8 Aug 2023 12:15:53 -0700 Subject: [PATCH 03/52] Update action.yml Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/action.yml | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index bfa87dd80d..15a8bf79f0 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -41,6 +41,12 @@ inputs: slsa-workflow-secret14: {} slsa-workflow-secret15: {} +outputs: + binaries-download-name: + description: "The name of binaries folder to download" + # NOTE: This is an "untrusted" value returned from the build. + value: "${{ steps.rng.outputs.random }}-binaries" + runs: using: "composite" steps: @@ -52,13 +58,47 @@ runs: uses: bazelbuild/setup-bazelisk@95c9bf48d0c570bb3e28e57108f3450cd67c1a44 # v2.0.0 - name: Setup Java + if: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} == 'true' id: java uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0 with: distribution: "${{ fromJson(inputs.slsa-workflow-inputs).user-java-distribution }}" java-version: "${{ fromJson(inputs.slsa-workflow-inputs).user-java-version }}" + - name: Check for Docker Image + id: docker + shell: bash + run: | + if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).docker-image }}" ]] + then + echo "No Docker Image provided. Will build without." + echo "USE_DOCKER=false" >> $GITHUB_ENV + else + echo "Docker image provided. Running build on Docker Image." + echo "USE_DOCKER=true" >> $GITHUB_ENV + fi + + - name: Build on Docker Image + if: env.USE_DOCKER == 'true' + env: + TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} + FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} + NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} + INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} + DOCKER_IMAGE: ${{ fromJson(inputs.slsa-workflow-inputs).docker-image }} + shell: bash + run: | + export TARGETS + export FLAGS + export NEEDS_RUNFILES + export INCLUDES_JAVA + + docker pull $DOCKER_IMAGE + curr_dir=$(basename "$(pwd)") + docker run --rm --env TARGETS=${TARGETS} --env FLAGS=${FLAGS} --env NEEDS_RUNFILES=${NEEDS_RUNFILES} --env INCLUDES_JAVA=${INCLUDES_JAVA} -v $PWD/../:/src -w /src $DOCKER_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" + - id: build + if: env.USE_DOCKER == 'false' env: TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From c51687659d59d8dca375409e318315f67066f0f8 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Thu, 10 Aug 2023 22:51:35 -0700 Subject: [PATCH 04/52] printf --> echo Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/rebuilder.sh | 112 +++++++++++++-------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 4cca841caf..4b5590c3ab 100644 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -78,25 +78,23 @@ verbose=0 # Boolean to trigger cleanup upon completion or failure. cleanup=0 - +# Disabled to stop triggering warnings about color env vars. + # shellcheck disable=SC2059 # Outputs the usage of the Rebuilder script for the two modes: # 1) Verify and Rebuild # 2) Rebuild only function usage() { if [[ $verify ]] then - # Disabled to stop triggering warnings about color env vars. - # shellcheck disable=SC2059 - printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}\n" - printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}\n" "$0" - printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}\n" - printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} \n" "$0" + echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" + echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" + echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" + echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " else - # shellcheck disable=SC2059 - printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}\n" - printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} \n" "$0" - printf "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}\n" - printf "${CYAN}Usage: %s ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}\n" "$0" + echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" + echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " + echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" + echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" fi } @@ -142,7 +140,7 @@ for ARG in "$@"; do if [[ ! ($returnValue) ]] then my_arg="$ARG" - printf "${RED}[ERROR] ${LIGHT_RED}%s is unrecognized${RESET}\n" "$my_arg" + echo "${RED}[ERROR] ${LIGHT_RED}$my_arg is unrecognized${RESET}" usage exit 1 fi @@ -156,19 +154,19 @@ done # Check if mandatory arguments for rebuild are not empty if [ -z "$artifact_path" ]; then - printf "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --artifact_path, is missing or empty${RESET}\n" + echo "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --artifact_path, is missing or empty${RESET}" usage exit 1 fi if [ -z "$prov_path" ]; then - printf "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --prov_path, is missing or empty${RESET}\n" + echo "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --prov_path, is missing or empty${RESET}" usage exit 1 fi if [ -z "$source_uri" ]; then - printf "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}\n" + echo "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}" usage exit 1 fi @@ -176,7 +174,7 @@ fi # Check if mandatory arguments for verification are not empty if [[ $verify -eq 1 && ( -z "$source_uri" || -z "$builder_id" ) ]] then - printf "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}\n" + echo "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}" usage exit 1 fi @@ -184,22 +182,22 @@ fi # Print received arguments (optional) if [[ $verbose -eq 1 ]] then - printf "${BLUE}โœ” Input Arguments Received:${RESET}\n" - printf "${CYAN}artifact_path: ${GREEN}%s${RESET}\n" "$artifact_path" - printf "${CYAN}prov_path: ${GREEN}%s${RESET}\n" "$prov_path" - printf "${CYAN}source_uri: ${GREEN}%s${RESET}\n" "$source_uri" + echo "${BLUE}โœ” Input Arguments Received:${RESET}" + echo "${CYAN}artifact_path: ${GREEN}$artifact_path${RESET}" + echo "${CYAN}prov_path: ${GREEN}$prov_path${RESET}" + echo "${CYAN}source_uri: ${GREEN}$source_uri${RESET}" if [ -n "$builder_id" ]; then - printf "${CYAN}builder_id: ${GREEN}%s${RESET}\n" "$builder_id" + echo "${CYAN}builder_id: ${GREEN}$builder_id${RESET}" fi if [ -n "$docker_image" ]; then - printf "${CYAN}docker_image: ${GREEN}%s${RESET}\n" "$docker_image" + echo "${CYAN}docker_image: ${GREEN}$docker_image${RESET}" fi - printf "${CYAN}verify: ${GREEN}%s${RESET}\n" "$verify" - printf "${CYAN}verbose: ${GREEN}%s${RESET}\n" "$verbose" - printf "${CYAN}cleanup: ${GREEN}%s${RESET}\n" "$cleanup" + echo "${CYAN}verify: ${GREEN}$verify${RESET}" + echo "${CYAN}verbose: ${GREEN}$verbose${RESET}" + echo "${CYAN}cleanup: ${GREEN}$cleanup${RESET}" echo "" fi @@ -217,7 +215,7 @@ then type_writer "โš ๏ธ---> To verify please remove the collision and try again" exit 1 else - printf "${CYAN}====================================================${RESET}\n" + echo "${CYAN}====================================================${RESET}" type_writer "๐Ÿ“ฅ---> The slsa-verifier repository is not cloned. Cloning..." git clone https://github.com/enteraga6/slsa-verifier fi @@ -231,7 +229,7 @@ then go run ./cli/slsa-verifier/ verify-artifact ../"$artifact_path" --provenance-path ../"$prov_path" --source-uri "$source_uri" --builder-id "$builder_id" cd .. - printf "${CYAN}====================================================${RESET}\n" + echo "${CYAN}====================================================${RESET}" echo "" fi @@ -257,10 +255,10 @@ done < <(cat "$prov_path" | jq -r '.dsseEnvelope.payload' | base64 -d | jq -r '. if [[ $verbose -eq 1 ]] then - printf "${PURPLE}โœ” Arguments Parsed from Provenance:${RESET}\n" + echo "${PURPLE}โœ” Arguments Parsed from Provenance:${RESET}" for key in "${!data[@]}" do - printf "${MAGENTA}$key: ${GREEN}%s${RESET}\n" "${data[$key]}" + echo "${MAGENTA}$key: ${GREEN}${data[$key]}${RESET}\" done echo "" fi @@ -299,16 +297,16 @@ done repo_name=$(basename "$source_uri") # Clone the source_uri repository to begin rebuild process if [ -d "$repo_name" ]; then - printf "${CYAN}====================================================${RESET}\n" + echo "${CYAN}====================================================${RESET}" type_writer "๐Ÿ“---> Source repository appears already." type_writer "โš ๏ธ---> To run rebuilder, fix collision by removing directory with name of $repo_name." exit 1 else - printf "${CYAN}====================================================${RESET}\n" + echo "${CYAN}====================================================${RESET}" type_writer "๐Ÿ‘---> Cloning the source repository..." echo "" git clone https://"$source_uri" - printf "${CYAN}====================================================${RESET}\n" + echo "${CYAN}====================================================${RESET}" echo "" fi @@ -341,9 +339,9 @@ fi ################################################ echo "" -printf "${CYAN}======================================================${RESET}\n" -printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} ๐Ÿ”จ Starting the Rebuild Process ๐Ÿ”จ ${RESET}${CYAN}|${RESET}\n" -printf "${CYAN}======================================================${RESET}\n" +echo "${CYAN}======================================================${RESET}" +echo "${CYAN}|${RESET}${YELLOW}${UNDERLINE} ๐Ÿ”จ Starting the Rebuild Process ๐Ÿ”จ ${RESET}${CYAN}|${RESET}" +echo "${CYAN}======================================================${RESET}" # Conditionals for docker images depend on if a Docker Image was use to build on Github. # If a Docker Image was not used to build on Github, then build locally. This is done to @@ -353,38 +351,38 @@ then cd - sudo docker pull "$DOCKER_IMAGE" echo "" - printf "${CYAN}====================================================${RESET}\n" + echo "${CYAN}======================================================${RESET}" type_writer "๐Ÿ”จ---> Rebuilding with Docker Image Environment..." # Mount docker image on this directory as workdir to gain access to script env - printf "${CYAN}====================================================${RESET}\n" + echo "${CYAN}======================================================${RESET}" echo "" sudo docker run --env repo_name="$repo_name" --env TARGETS="${TARGETS}" --env FLAGS="${FLAGS}" --env NEEDS_RUNFILES="${NEEDS_RUNFILES}" --env INCLUDES_JAVA="${INCLUDES_JAVA}" --rm -v "$PWD":/workdir -w /workdir "$DOCKER_IMAGE" /bin/sh -c "cd $repo_name && ./../build.sh" echo "" - printf "${CYAN}=============================================${RESET}\n" - printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}\n" - printf "${CYAN}=============================================${RESET}\n" + echo "${CYAN}======================================================${RESET}" + echo "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}" + echo "${CYAN}======================================================${RESET}" echo "" else if [[ -n "$docker_image" ]] then # Warning message for the users if their artifact was not built with a Docker Image, but a Docker Image was provided at command. - printf "${RED}[Warning] ${LIGHT_RED}Docker Image, %s, provided, but artifact was not originally built on Docker Image${RESET}\n" "$docker_image" + echo "${RED}[Warning] ${LIGHT_RED}Docker Image, $docker_image, provided, but artifact was not originally built on Docker Image${RESET}" else echo "" # This is just for style. fi # Run the build script locally without a docker image. - printf "${CYAN}=============================================${RESET}\n" + echo "${CYAN}======================================================${RESET}" type_writer "๐Ÿ’ป---> Rebuilding with local environment..." - printf "${CYAN}=============================================${RESET}\n" + echo "${CYAN}======================================================${RESET}" echo "" # shellcheck source=../build.sh source ../build.sh echo "" - printf "${CYAN}=============================================${RESET}\n" + echo "${CYAN}======================================================${RESET}" printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}\n" - printf "${CYAN}=============================================${RESET}\n" + echo "${CYAN}======================================================${RESET}" echo "" fi @@ -454,21 +452,21 @@ fi ################################################ if [[ "$orig_checksum" == "$rebuilt_checksum" ]] -then - printf "${GREEN}Checksum is the ${BOLD}${UNDERLINE}same${RESET}${GREEN} for the original and rebuilt artifact!${RESET}\n" - printf "${GREEN}โœ… This build is ${BOLD}${UNDERLINE}reproducible!${RESET} โœ… \n" + then + echo "${GREEN}Checksum is the ${BOLD}${UNDERLINE}same${RESET}${GREEN} for the original and rebuilt artifact!${RESET}" + echo "${GREEN}โœ… This build is ${BOLD}${UNDERLINE}reproducible!${RESET} โœ…" echo "" - printf "${GREEN}%s${RESET} = Original Checksum${RESET}\n" "$orig_checksum" - printf "${GREEN}%s${RESET} = Rebuilt Checksum${RESET}\n" "$rebuilt_checksum" + echo "${GREEN}$orig_checksum${RESET} = Original Checksum" + echo "${GREEN}$rebuilt_checksum${RESET} = Rebuilt Checksum" echo "" -else - printf "${BOLD_RED_BG}Checksum is ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} the same for the original and rebuilt artifact!${RESET}\n" - printf "${BOLD_RED_BG} โš ๏ธ This build was ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} able to be reproduced! โš ๏ธ ${RESET}\n" + else + echo "${BOLD_RED_BG}Checksum is ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} the same for the original and rebuilt artifact!${RESET}" + echo "${BOLD_RED_BG} โš ๏ธ This build was ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} able to be reproduced! โš ๏ธ ${RESET}" echo "" - printf "${RED}%s${RESET} = Original Checksum\n" "$orig_checksum" - printf "${RED}%s${RESET} = Rebuilt Checksum\n" "$rebuilt_checksum" + echo "${RED}$orig_checksum${RESET} = Original Checksum" + echo "${RED}$rebuilt_checksum${RESET} = Rebuilt Checksum" echo "" -fi + fi if [[ cleanup -eq 1 ]] From 9f679ee4f33cbdecdacd34f2a6a1a20ba655f059 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 06:09:44 +0000 Subject: [PATCH 05/52] finish echo conversion Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 111 ++++++++++++++------------- 1 file changed, 57 insertions(+), 54 deletions(-) mode change 100644 => 100755 internal/builders/bazel/rebuilder.sh diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh old mode 100644 new mode 100755 index 4b5590c3ab..f236657990 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -86,15 +86,15 @@ cleanup=0 function usage() { if [[ $verify ]] then - echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" - echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" - echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" - echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " + echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" + echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " else - echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" - echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " - echo "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" - echo "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" + echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " + echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" fi } @@ -140,7 +140,7 @@ for ARG in "$@"; do if [[ ! ($returnValue) ]] then my_arg="$ARG" - echo "${RED}[ERROR] ${LIGHT_RED}$my_arg is unrecognized${RESET}" + echo -e "${RED}[ERROR] ${LIGHT_RED}$my_arg is unrecognized${RESET}" usage exit 1 fi @@ -154,19 +154,19 @@ done # Check if mandatory arguments for rebuild are not empty if [ -z "$artifact_path" ]; then - echo "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --artifact_path, is missing or empty${RESET}" + echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --artifact_path, is missing or empty${RESET}" usage exit 1 fi if [ -z "$prov_path" ]; then - echo "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --prov_path, is missing or empty${RESET}" + echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --prov_path, is missing or empty${RESET}" usage exit 1 fi if [ -z "$source_uri" ]; then - echo "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}" + echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}" usage exit 1 fi @@ -174,7 +174,7 @@ fi # Check if mandatory arguments for verification are not empty if [[ $verify -eq 1 && ( -z "$source_uri" || -z "$builder_id" ) ]] then - echo "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}" + echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}" usage exit 1 fi @@ -182,22 +182,22 @@ fi # Print received arguments (optional) if [[ $verbose -eq 1 ]] then - echo "${BLUE}โœ” Input Arguments Received:${RESET}" - echo "${CYAN}artifact_path: ${GREEN}$artifact_path${RESET}" - echo "${CYAN}prov_path: ${GREEN}$prov_path${RESET}" - echo "${CYAN}source_uri: ${GREEN}$source_uri${RESET}" + echo -e "${BLUE}โœ” Input Arguments Received:${RESET}" + echo -e "${CYAN}artifact_path: ${GREEN}$artifact_path${RESET}" + echo -e "${CYAN}prov_path: ${GREEN}$prov_path${RESET}" + echo -e "${CYAN}source_uri: ${GREEN}$source_uri${RESET}" if [ -n "$builder_id" ]; then - echo "${CYAN}builder_id: ${GREEN}$builder_id${RESET}" + echo -e "${CYAN}builder_id: ${GREEN}$builder_id${RESET}" fi if [ -n "$docker_image" ]; then - echo "${CYAN}docker_image: ${GREEN}$docker_image${RESET}" + echo -e "${CYAN}docker_image: ${GREEN}$docker_image${RESET}" fi - echo "${CYAN}verify: ${GREEN}$verify${RESET}" - echo "${CYAN}verbose: ${GREEN}$verbose${RESET}" - echo "${CYAN}cleanup: ${GREEN}$cleanup${RESET}" + echo -e "${CYAN}verify: ${GREEN}$verify${RESET}" + echo -e "${CYAN}verbose: ${GREEN}$verbose${RESET}" + echo -e "${CYAN}cleanup: ${GREEN}$cleanup${RESET}" echo "" fi @@ -215,7 +215,7 @@ then type_writer "โš ๏ธ---> To verify please remove the collision and try again" exit 1 else - echo "${CYAN}====================================================${RESET}" + echo -e "${CYAN}====================================================${RESET}" type_writer "๐Ÿ“ฅ---> The slsa-verifier repository is not cloned. Cloning..." git clone https://github.com/enteraga6/slsa-verifier fi @@ -229,7 +229,7 @@ then go run ./cli/slsa-verifier/ verify-artifact ../"$artifact_path" --provenance-path ../"$prov_path" --source-uri "$source_uri" --builder-id "$builder_id" cd .. - echo "${CYAN}====================================================${RESET}" + echo -e "${CYAN}====================================================${RESET}" echo "" fi @@ -255,10 +255,10 @@ done < <(cat "$prov_path" | jq -r '.dsseEnvelope.payload' | base64 -d | jq -r '. if [[ $verbose -eq 1 ]] then - echo "${PURPLE}โœ” Arguments Parsed from Provenance:${RESET}" + echo -e "${PURPLE}โœ” Arguments Parsed from Provenance:${RESET}" for key in "${!data[@]}" do - echo "${MAGENTA}$key: ${GREEN}${data[$key]}${RESET}\" + echo -e "${MAGENTA}$key: ${GREEN}${data[$key]}${RESET}" done echo "" fi @@ -297,16 +297,16 @@ done repo_name=$(basename "$source_uri") # Clone the source_uri repository to begin rebuild process if [ -d "$repo_name" ]; then - echo "${CYAN}====================================================${RESET}" + echo -e "${CYAN}====================================================${RESET}" type_writer "๐Ÿ“---> Source repository appears already." type_writer "โš ๏ธ---> To run rebuilder, fix collision by removing directory with name of $repo_name." exit 1 else - echo "${CYAN}====================================================${RESET}" + echo -e "${CYAN}====================================================${RESET}" type_writer "๐Ÿ‘---> Cloning the source repository..." echo "" git clone https://"$source_uri" - echo "${CYAN}====================================================${RESET}" + echo -e "${CYAN}====================================================${RESET}" echo "" fi @@ -339,9 +339,9 @@ fi ################################################ echo "" -echo "${CYAN}======================================================${RESET}" -echo "${CYAN}|${RESET}${YELLOW}${UNDERLINE} ๐Ÿ”จ Starting the Rebuild Process ๐Ÿ”จ ${RESET}${CYAN}|${RESET}" -echo "${CYAN}======================================================${RESET}" +echo -e "${CYAN}======================================================${RESET}" +echo -e "${CYAN}|${RESET}${YELLOW}${UNDERLINE} ๐Ÿ”จ Starting the Rebuild Process ๐Ÿ”จ ${RESET}${CYAN}|${RESET}" +echo -e "${CYAN}======================================================${RESET}" # Conditionals for docker images depend on if a Docker Image was use to build on Github. # If a Docker Image was not used to build on Github, then build locally. This is done to @@ -351,38 +351,38 @@ then cd - sudo docker pull "$DOCKER_IMAGE" echo "" - echo "${CYAN}======================================================${RESET}" + echo -e "${CYAN}======================================================${RESET}" type_writer "๐Ÿ”จ---> Rebuilding with Docker Image Environment..." # Mount docker image on this directory as workdir to gain access to script env - echo "${CYAN}======================================================${RESET}" + echo -e "${CYAN}======================================================${RESET}" echo "" sudo docker run --env repo_name="$repo_name" --env TARGETS="${TARGETS}" --env FLAGS="${FLAGS}" --env NEEDS_RUNFILES="${NEEDS_RUNFILES}" --env INCLUDES_JAVA="${INCLUDES_JAVA}" --rm -v "$PWD":/workdir -w /workdir "$DOCKER_IMAGE" /bin/sh -c "cd $repo_name && ./../build.sh" echo "" - echo "${CYAN}======================================================${RESET}" - echo "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}" - echo "${CYAN}======================================================${RESET}" + echo -e "${CYAN}======================================================${RESET}" + echo -e "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}" + echo -e "${CYAN}======================================================${RESET}" echo "" else if [[ -n "$docker_image" ]] then # Warning message for the users if their artifact was not built with a Docker Image, but a Docker Image was provided at command. - echo "${RED}[Warning] ${LIGHT_RED}Docker Image, $docker_image, provided, but artifact was not originally built on Docker Image${RESET}" + echo -e "${RED}[Warning] ${LIGHT_RED}Docker Image, $docker_image, provided, but artifact was not originally built on Docker Image${RESET}" else echo "" # This is just for style. fi # Run the build script locally without a docker image. - echo "${CYAN}======================================================${RESET}" + echo -e "${CYAN}======================================================${RESET}" type_writer "๐Ÿ’ป---> Rebuilding with local environment..." - echo "${CYAN}======================================================${RESET}" + echo -e "${CYAN}======================================================${RESET}" echo "" # shellcheck source=../build.sh source ../build.sh echo "" - echo "${CYAN}======================================================${RESET}" + echo -e "${CYAN}======================================================${RESET}" printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}\n" - echo "${CYAN}======================================================${RESET}" + echo -e "${CYAN}======================================================${RESET}" echo "" fi @@ -453,18 +453,18 @@ fi if [[ "$orig_checksum" == "$rebuilt_checksum" ]] then - echo "${GREEN}Checksum is the ${BOLD}${UNDERLINE}same${RESET}${GREEN} for the original and rebuilt artifact!${RESET}" - echo "${GREEN}โœ… This build is ${BOLD}${UNDERLINE}reproducible!${RESET} โœ…" + echo -e "${GREEN}Checksum is the ${BOLD}${UNDERLINE}same${RESET}${GREEN} for the original and rebuilt artifact!${RESET}" + echo -e "${GREEN}โœ… This build is ${BOLD}${UNDERLINE}reproducible!${RESET} โœ…" echo "" - echo "${GREEN}$orig_checksum${RESET} = Original Checksum" - echo "${GREEN}$rebuilt_checksum${RESET} = Rebuilt Checksum" + echo -e "${GREEN}$orig_checksum${RESET} = Original Checksum" + echo -e "${GREEN}$rebuilt_checksum${RESET} = Rebuilt Checksum" echo "" else - echo "${BOLD_RED_BG}Checksum is ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} the same for the original and rebuilt artifact!${RESET}" - echo "${BOLD_RED_BG} โš ๏ธ This build was ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} able to be reproduced! โš ๏ธ ${RESET}" + echo -e "${BOLD_RED_BG}Checksum is ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} the same for the original and rebuilt artifact!${RESET}" + echo -e "${BOLD_RED_BG} โš ๏ธ This build was ${BOLD}${UNDERLINE}NOT${RESET}${BOLD_RED_BG} able to be reproduced! โš ๏ธ ${RESET}" echo "" - echo "${RED}$orig_checksum${RESET} = Original Checksum" - echo "${RED}$rebuilt_checksum${RESET} = Rebuilt Checksum" + echo -e "${RED}$orig_checksum${RESET} = Original Checksum" + echo -e "${RED}$rebuilt_checksum${RESET} = Rebuilt Checksum" echo "" fi @@ -474,10 +474,13 @@ then # If there are runfiles or if the artifacts are Java, then each artifact # has its own directory, so you need to exit out of it first. - if [[ "${NEEDS_RUNFILES}" == "true" || "${INCLUDES_JAVA}" == "true" ]] - then - cd .. - fi + + # TODO TEST FUNCTIONALITY OF CLEANUP + + # if [[ "${NEEDS_RUNFILES}" == "true" || "${INCLUDES_JAVA}" == "true" ]] + # then + # cd .. + # fi # Current position is bazel_builder_dir/$repo_name/$binaries_dir, # and to clean up need to be in /bazel. From 5f64a762e941ebe438bed92441a65d0193187e73 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 06:12:49 +0000 Subject: [PATCH 06/52] shell check Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index f236657990..bea19585d9 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -377,11 +377,11 @@ else echo -e "${CYAN}======================================================${RESET}" echo "" - # shellcheck source=../build.sh + # shellcheck source=./../build.sh source ../build.sh echo "" echo -e "${CYAN}======================================================${RESET}" - printf "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}\n" + echo -e "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}" echo -e "${CYAN}======================================================${RESET}" echo "" fi From 20e0b20622a1faf297a3d58be95df46b4c05b2ad Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 06:31:29 +0000 Subject: [PATCH 07/52] source shellcheck fix att #2 Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index bea19585d9..3cba46497b 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -377,7 +377,7 @@ else echo -e "${CYAN}======================================================${RESET}" echo "" - # shellcheck source=./../build.sh + # shellcheck source=./build.sh source ../build.sh echo "" echo -e "${CYAN}======================================================${RESET}" From 3c26e328859979c6cacda4aaa4b1c9a7eb059ce6 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 06:34:30 +0000 Subject: [PATCH 08/52] source shellcheck fix att 3 Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 3cba46497b..b7875df992 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -377,7 +377,8 @@ else echo -e "${CYAN}======================================================${RESET}" echo "" - # shellcheck source=./build.sh + # To fix shell check + # shellcheck source=/dev/null source ../build.sh echo "" echo -e "${CYAN}======================================================${RESET}" From e6f8366f9defeb82f332738b17ddd99e73c060a7 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 06:43:26 +0000 Subject: [PATCH 09/52] cat abuse shellcheck Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index b7875df992..7981520beb 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -249,7 +249,7 @@ declare -A data # and the value is the value that the user inputted. Pipe value to @text, to deal with booleans. while IFS='=' read -r key value; do data["$key"]="$value" -done < <(cat "$prov_path" | jq -r '.dsseEnvelope.payload' | base64 -d | jq -r '.predicate.buildDefinition.externalParameters.inputs | to_entries | .[] | .key + "=" + (.value | @text)') +done < <(<"$prov_path" jq -r '.dsseEnvelope.payload' | base64 -d | jq -r '.predicate.buildDefinition.externalParameters.inputs | to_entries | .[] | .key + "=" + (.value | @text)') # Todo: Style Env Vars Later From be49f2ef585f40652dfeb189a61cd76b39967335 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Thu, 10 Aug 2023 23:45:39 -0700 Subject: [PATCH 10/52] binaries --> artifacts Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- .github/workflows/builder_bazel_slsa3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builder_bazel_slsa3.yml b/.github/workflows/builder_bazel_slsa3.yml index 6e40b001d8..eb0508d3fc 100644 --- a/.github/workflows/builder_bazel_slsa3.yml +++ b/.github/workflows/builder_bazel_slsa3.yml @@ -88,7 +88,7 @@ on: Users should verify the download against this digest to prevent tampering. value: ${{ jobs.slsa-run.outputs.attestations-download-sha256 }} - binaries-download-name: + artifacts-download-name: description: > The name of the folder containing the built artifacts. There is a random hash at the beginning of it in form -binaries to avoid collisions. From 3a5a99f05566ed5c210d7a98dd3e92fcbf42fb85 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Thu, 10 Aug 2023 23:48:35 -0700 Subject: [PATCH 11/52] binaries --> artifacts Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 15a8bf79f0..1be04eb310 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -42,7 +42,7 @@ inputs: slsa-workflow-secret15: {} outputs: - binaries-download-name: + artifacts-download-name: description: "The name of binaries folder to download" # NOTE: This is an "untrusted" value returned from the build. value: "${{ steps.rng.outputs.random }}-binaries" From d033ee736f970eeeeef3050979c31320526c7aff Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Thu, 10 Aug 2023 23:55:32 -0700 Subject: [PATCH 12/52] Update action.yml Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 1be04eb310..1ca3dc2f57 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -46,6 +46,9 @@ outputs: description: "The name of binaries folder to download" # NOTE: This is an "untrusted" value returned from the build. value: "${{ steps.rng.outputs.random }}-binaries" + artifacts-download-sha256: + description: "SHA256 of the uploaded tarball of artifacts." + value: ${{ steps.generate-artifacts.outputs.sha256 }} runs: using: "composite" From ce6082dc33e47d939451c14736aaf962b1fd14ff Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Thu, 10 Aug 2023 23:57:25 -0700 Subject: [PATCH 13/52] add sha256 download output Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- .github/workflows/builder_bazel_slsa3.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/builder_bazel_slsa3.yml b/.github/workflows/builder_bazel_slsa3.yml index eb0508d3fc..37acd68f03 100644 --- a/.github/workflows/builder_bazel_slsa3.yml +++ b/.github/workflows/builder_bazel_slsa3.yml @@ -92,7 +92,11 @@ on: description: > The name of the folder containing the built artifacts. There is a random hash at the beginning of it in form -binaries to avoid collisions. - value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).binaries-download-name }} + value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).artifacts-download-name }} + + artifacts-download-sha256: + description: "SHA256 of the uploaded tarball of built artifacts." + value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).artifacts-download-sha256 }} jobs: slsa-setup: From 849d25b108e925d05f65ebc693be8184cf792e3e Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Thu, 10 Aug 2023 23:58:56 -0700 Subject: [PATCH 14/52] Update action.yml Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/action.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 1ca3dc2f57..af4a445780 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -45,7 +45,7 @@ outputs: artifacts-download-name: description: "The name of binaries folder to download" # NOTE: This is an "untrusted" value returned from the build. - value: "${{ steps.rng.outputs.random }}-binaries" + value: "${{ steps.rng.outputs.random }}-artifacts" artifacts-download-sha256: description: "SHA256 of the uploaded tarball of artifacts." value: ${{ steps.generate-artifacts.outputs.sha256 }} @@ -91,11 +91,6 @@ runs: DOCKER_IMAGE: ${{ fromJson(inputs.slsa-workflow-inputs).docker-image }} shell: bash run: | - export TARGETS - export FLAGS - export NEEDS_RUNFILES - export INCLUDES_JAVA - docker pull $DOCKER_IMAGE curr_dir=$(basename "$(pwd)") docker run --rm --env TARGETS=${TARGETS} --env FLAGS=${FLAGS} --env NEEDS_RUNFILES=${NEEDS_RUNFILES} --env INCLUDES_JAVA=${INCLUDES_JAVA} -v $PWD/../:/src -w /src $DOCKER_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" @@ -120,7 +115,7 @@ runs: id: generate-artifacts uses: slsa-framework/slsa-github-generator/.github/actions/secure-upload-folder@v1.8.0 with: - name: "${{ steps.rng.outputs.random }}-binaries" + name: "${{ steps.rng.outputs.random }}-artifacts" path: "./bazel_builder_binaries_to_upload_to_gh_7bc972367cb286b7f36ab4457f06e369" # path-to-artifact(s) - name: Echo statement From 9a27d429c5196260d4d17a9a6f1baca20c644642 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:08:18 +0000 Subject: [PATCH 15/52] set -u and used Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 29 +++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 7981520beb..b18c87e07a 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: -u not set to check for empty variables from parse arguments function. -set -eo pipefail - -# Disabled to stop triggering warnings about color env vars. +set -euo pipefail # This directory is where the rebuilt artifacts will be stored. It is made upon # running the rebuilder. The long name is to avoid potential collisions. @@ -78,8 +75,7 @@ verbose=0 # Boolean to trigger cleanup upon completion or failure. cleanup=0 -# Disabled to stop triggering warnings about color env vars. - # shellcheck disable=SC2059 + # Outputs the usage of the Rebuilder script for the two modes: # 1) Verify and Rebuild # 2) Rebuild only @@ -153,26 +149,26 @@ done ################################################ # Check if mandatory arguments for rebuild are not empty -if [ -z "$artifact_path" ]; then +if [ -z ${artifact_path:-} ]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --artifact_path, is missing or empty${RESET}" usage exit 1 fi -if [ -z "$prov_path" ]; then +if [ -z ${prov_path:-} ]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --prov_path, is missing or empty${RESET}" usage exit 1 fi -if [ -z "$source_uri" ]; then +if [ -z {$source_uri:-} ]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}" usage exit 1 fi # Check if mandatory arguments for verification are not empty -if [[ $verify -eq 1 && ( -z "$source_uri" || -z "$builder_id" ) ]] +if [[ $verify -eq 1 && ( -z ${source_uri:-} || -z {$builder_id:-} ) ]] then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}" usage @@ -187,11 +183,11 @@ then echo -e "${CYAN}prov_path: ${GREEN}$prov_path${RESET}" echo -e "${CYAN}source_uri: ${GREEN}$source_uri${RESET}" - if [ -n "$builder_id" ]; then + if [ -n ${builder_id:-} ]; then echo -e "${CYAN}builder_id: ${GREEN}$builder_id${RESET}" fi - if [ -n "$docker_image" ]; then + if [ -n ${docker_image:-} ]; then echo -e "${CYAN}docker_image: ${GREEN}$docker_image${RESET}" fi @@ -346,7 +342,7 @@ echo -e "${CYAN}======================================================${RESET}" # Conditionals for docker images depend on if a Docker Image was use to build on Github. # If a Docker Image was not used to build on Github, then build locally. This is done to # ensure consistent build environment between both platforms. -if [[ -n $DOCKER_IMAGE ]] +if [[ -n ${DOCKER_IMAGE:-} ]] then cd - sudo docker pull "$DOCKER_IMAGE" @@ -363,7 +359,7 @@ then echo -e "${CYAN}======================================================${RESET}" echo "" else - if [[ -n "$docker_image" ]] + if [[ -n ${docker_image:-} ]] then # Warning message for the users if their artifact was not built with a Docker Image, but a Docker Image was provided at command. echo -e "${RED}[Warning] ${LIGHT_RED}Docker Image, $docker_image, provided, but artifact was not originally built on Docker Image${RESET}" @@ -387,12 +383,9 @@ else echo "" fi -# To avoid unbound variable after build script which sets -euo. -set +u - # If Docker Image was used to build on Github, we need to cd into repo # to access the binaries directory. -if [[ -n $DOCKER_IMAGE ]] +if [[ -n ${DOCKER_IMAGE:-} ]] then cd "$repo_name" fi From 503ae6a26d41253044805a1624e727f130c591c1 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:11:58 +0000 Subject: [PATCH 16/52] use offical slsa-verifier repo Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index b18c87e07a..86839345ce 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -213,7 +213,7 @@ then else echo -e "${CYAN}====================================================${RESET}" type_writer "๐Ÿ“ฅ---> The slsa-verifier repository is not cloned. Cloning..." - git clone https://github.com/enteraga6/slsa-verifier + git clone https://github.com/slsa-framework/slsa-verifier fi # Change directory to the slsa-verifier directory From cde809ada47e5c48981acbab12a9c56cfbd4ebcf Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Fri, 11 Aug 2023 00:13:50 -0700 Subject: [PATCH 17/52] Update internal/builders/bazel/rebuilder.sh Co-authored-by: Ian Lewis Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/rebuilder.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 86839345ce..43546dd1f8 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -80,8 +80,7 @@ cleanup=0 # 1) Verify and Rebuild # 2) Rebuild only function usage() { - if [[ $verify ]] - then + if [[ $verify ]]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" From a5936e97a8116712d6d8e3a273af2ef984c0ad1c Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Fri, 11 Aug 2023 00:14:43 -0700 Subject: [PATCH 18/52] Update internal/builders/bazel/rebuilder.sh Co-authored-by: Ian Lewis Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/rebuilder.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 43546dd1f8..e8b815a9e2 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -462,8 +462,7 @@ if [[ "$orig_checksum" == "$rebuilt_checksum" ]] fi -if [[ cleanup -eq 1 ]] -then +if [[ cleanup -eq 1 ]]; then # If there are runfiles or if the artifacts are Java, then each artifact # has its own directory, so you need to exit out of it first. From fe3b22e449f11d62f185475c27761f023303e620 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:19:01 +0000 Subject: [PATCH 19/52] nits: indents and consistency Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 45 +++++++++++++--------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 86839345ce..17eb2a2d3a 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -422,20 +422,19 @@ then # Copy the entire directory, including the run script. cp -R ./"$run_script_name" ./../../"$rebuilt_artifacts_dir"/ else - if [[ "${NEEDS_RUNFILES}" == "true" ]] - then + if [[ "${NEEDS_RUNFILES}" == "true" ]]; then # For non-java targets with runfiles. - cd $binaries_dir/ - rebuilt_checksum=$(sha256sum ./"$artifact_name"/"$artifact_name" | awk '{ print $1 }') + cd $binaries_dir/ + rebuilt_checksum=$(sha256sum ./"$artifact_name"/"$artifact_name" | awk '{ print $1 }') - # Copy entire directory, including the runfiles. - cp -R ./"$artifact_name" ./../../"$rebuilt_artifacts_dir"/ + # Copy entire directory, including the runfiles. + cp -R ./"$artifact_name" ./../../"$rebuilt_artifacts_dir"/ else - # For files withouts runfiles. - cd $binaries_dir - rebuilt_checksum=$(sha256sum "$artifact_name" | awk '{ print $1 }') + # For files withouts runfiles. + cd $binaries_dir + rebuilt_checksum=$(sha256sum "$artifact_name" | awk '{ print $1 }') - cp "$artifact_name" ./../../$rebuilt_artifacts_dir/ + cp "$artifact_name" ./../../$rebuilt_artifacts_dir/ fi fi @@ -445,8 +444,7 @@ fi # # ################################################ -if [[ "$orig_checksum" == "$rebuilt_checksum" ]] - then +if [[ "$orig_checksum" == "$rebuilt_checksum" ]]; then echo -e "${GREEN}Checksum is the ${BOLD}${UNDERLINE}same${RESET}${GREEN} for the original and rebuilt artifact!${RESET}" echo -e "${GREEN}โœ… This build is ${BOLD}${UNDERLINE}reproducible!${RESET} โœ…" echo "" @@ -466,20 +464,19 @@ if [[ "$orig_checksum" == "$rebuilt_checksum" ]] if [[ cleanup -eq 1 ]] then - # If there are runfiles or if the artifacts are Java, then each artifact - # has its own directory, so you need to exit out of it first. + # If there are runfiles or if the artifacts are Java, then each artifact + # has its own directory, so you need to exit out of it first. - # TODO TEST FUNCTIONALITY OF CLEANUP + # TODO TEST FUNCTIONALITY OF CLEANUP - # if [[ "${NEEDS_RUNFILES}" == "true" || "${INCLUDES_JAVA}" == "true" ]] - # then - # cd .. - # fi + # if [[ "${NEEDS_RUNFILES}" == "true" || "${INCLUDES_JAVA}" == "true" ]]; then + # cd .. + # fi - # Current position is bazel_builder_dir/$repo_name/$binaries_dir, - # and to clean up need to be in /bazel. - cd ../.. + # Current position is bazel_builder_dir/$repo_name/$binaries_dir, + # and to clean up need to be in /bazel. + cd ../.. - # Now cleanup of verifier and cloned $repo_name. - cleanup + # Now cleanup of verifier and cloned $repo_name. + cleanup fi From 46e7587b6f0ede899629e166fb4dbf73f94757a2 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Fri, 11 Aug 2023 00:20:05 -0700 Subject: [PATCH 20/52] Update internal/builders/bazel/rebuilder.sh Co-authored-by: Ian Lewis Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/rebuilder.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index a63fdd93a3..1e33729eac 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -202,8 +202,7 @@ fi # # ################################################ -if [[ $verify -eq 1 ]] -then +if [[ $verify -eq 1 ]]; then # Clone the slsa-verifier repository if [ -d "slsa-verifier" ]; then type_writer "๐Ÿ“---> The slsa-verifier repository is already cloned." From 872e2a972878b5c7fc6db95397ecb3e0b6da54ff Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:27:17 +0000 Subject: [PATCH 21/52] nits: consistency Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 33 ++++++++++------------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index a63fdd93a3..9403970376 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -132,8 +132,7 @@ function cleanup() { for ARG in "$@"; do returnValue=$? process_argument "$ARG" - if [[ ! ($returnValue) ]] - then + if [[ ! ($returnValue) ]]; then my_arg="$ARG" echo -e "${RED}[ERROR] ${LIGHT_RED}$my_arg is unrecognized${RESET}" usage @@ -167,16 +166,14 @@ if [ -z {$source_uri:-} ]; then fi # Check if mandatory arguments for verification are not empty -if [[ $verify -eq 1 && ( -z ${source_uri:-} || -z {$builder_id:-} ) ]] -then +if [[ $verify -eq 1 && ( -z ${source_uri:-} || -z {$builder_id:-} ) ]]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}" usage exit 1 fi # Print received arguments (optional) -if [[ $verbose -eq 1 ]] -then +if [[ $verbose -eq 1 ]]; then echo -e "${BLUE}โœ” Input Arguments Received:${RESET}" echo -e "${CYAN}artifact_path: ${GREEN}$artifact_path${RESET}" echo -e "${CYAN}prov_path: ${GREEN}$prov_path${RESET}" @@ -248,11 +245,9 @@ done < <(<"$prov_path" jq -r '.dsseEnvelope.payload' | base64 -d | jq -r '.predi # Todo: Style Env Vars Later -if [[ $verbose -eq 1 ]] -then +if [[ $verbose -eq 1 ]]; then echo -e "${PURPLE}โœ” Arguments Parsed from Provenance:${RESET}" - for key in "${!data[@]}" - do + for key in "${!data[@]}"; do echo -e "${MAGENTA}$key: ${GREEN}${data[$key]}${RESET}" done echo "" @@ -312,8 +307,7 @@ cd "$repo_name" # avoid triggering unbound variable error. if [[ "${INCLUDES_JAVA}" == "true" ]] then - if [[ ! -v JAVA_HOME || -z "${JAVA_HOME}" ]] - then + if [[ ! -v JAVA_HOME || -z "${JAVA_HOME}" ]]; then # if JAVA_HOME is empty, set to jdk bin path from $(which java) if java_path=$(which java); then JAVA_HOME="$(dirname "$(dirname "${java_path}")")" @@ -341,8 +335,7 @@ echo -e "${CYAN}======================================================${RESET}" # Conditionals for docker images depend on if a Docker Image was use to build on Github. # If a Docker Image was not used to build on Github, then build locally. This is done to # ensure consistent build environment between both platforms. -if [[ -n ${DOCKER_IMAGE:-} ]] -then +if [[ -n ${DOCKER_IMAGE:-} ]]; then cd - sudo docker pull "$DOCKER_IMAGE" echo "" @@ -358,8 +351,7 @@ then echo -e "${CYAN}======================================================${RESET}" echo "" else - if [[ -n ${docker_image:-} ]] - then + if [[ -n ${docker_image:-} ]]; then # Warning message for the users if their artifact was not built with a Docker Image, but a Docker Image was provided at command. echo -e "${RED}[Warning] ${LIGHT_RED}Docker Image, $docker_image, provided, but artifact was not originally built on Docker Image${RESET}" else @@ -384,8 +376,7 @@ fi # If Docker Image was used to build on Github, we need to cd into repo # to access the binaries directory. -if [[ -n ${DOCKER_IMAGE:-} ]] -then +if [[ -n ${DOCKER_IMAGE:-} ]]; then cd "$repo_name" fi @@ -396,8 +387,7 @@ fi ################################################ # Obtain the name of the artifact -if [[ $artifact_path == */* ]] -then +if [[ $artifact_path == */* ]]; then artifact_name=$(basename "$artifact_path") else artifact_name=$artifact_path @@ -410,8 +400,7 @@ unset rebuilt_checksum # Makes sure it is empty before assigning. # The binaries folder contains different directories for the its artifacts and # the artifacts runfiles. Obtain the rebuilt binaries and copy them to the # path at root before cleaning up and deleting the repo. -if [[ "$artifact_name" == *"_deploy.jar"* ]] -then +if [[ "$artifact_name" == *"_deploy.jar"* ]]; then # Uses _deploy.jar as a field seperator and grabs the field before it. # Directory of Java artifacts is same as run script name. run_script_name=$(echo "$artifact_name" | awk -F'_deploy.jar' '{print $1}') From 6441936d50d9d6e5806bb28c771f71198d88a335 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:38:58 +0000 Subject: [PATCH 22/52] shellcheck Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 538b065301..48852da3b1 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -159,14 +159,14 @@ if [ -z ${prov_path:-} ]; then exit 1 fi -if [ -z {$source_uri:-} ]; then +if [ -z ${source_uri:-} ]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}" usage exit 1 fi # Check if mandatory arguments for verification are not empty -if [[ $verify -eq 1 && ( -z ${source_uri:-} || -z {$builder_id:-} ) ]]; then +if [[ $verify -eq 1 && ( -z ${source_uri:-} || -z ${builder_id:-} ) ]]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory arguments for verification missing or empty${RESET}" usage exit 1 @@ -179,11 +179,11 @@ if [[ $verbose -eq 1 ]]; then echo -e "${CYAN}prov_path: ${GREEN}$prov_path${RESET}" echo -e "${CYAN}source_uri: ${GREEN}$source_uri${RESET}" - if [ -n ${builder_id:-} ]; then + if [[ -n ${builder_id:-} ]]; then echo -e "${CYAN}builder_id: ${GREEN}$builder_id${RESET}" fi - if [ -n ${docker_image:-} ]; then + if [[ -n ${docker_image:-} ]]; then echo -e "${CYAN}docker_image: ${GREEN}$docker_image${RESET}" fi From 4a4769acbb465fc714090a8162156f41238d9110 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:40:59 +0000 Subject: [PATCH 23/52] double quotes Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 48852da3b1..de9a043cff 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -147,19 +147,19 @@ done ################################################ # Check if mandatory arguments for rebuild are not empty -if [ -z ${artifact_path:-} ]; then +if [ -z "${artifact_path:-}" ]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --artifact_path, is missing or empty${RESET}" usage exit 1 fi -if [ -z ${prov_path:-} ]; then +if [ -z "${prov_path:-}" ]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --prov_path, is missing or empty${RESET}" usage exit 1 fi -if [ -z ${source_uri:-} ]; then +if [ -z "${source_uri:-}" ]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Mandatory argument for rebuild, --source_uri, is missing or empty${RESET}" usage exit 1 From 62e616c1b0865376c5cc3c9b1706febe8f8c21e9 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:42:15 +0000 Subject: [PATCH 24/52] lint Signed-off-by: Noah Elzner --- .github/workflows/builder_bazel_slsa3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builder_bazel_slsa3.yml b/.github/workflows/builder_bazel_slsa3.yml index 37acd68f03..4a607a5648 100644 --- a/.github/workflows/builder_bazel_slsa3.yml +++ b/.github/workflows/builder_bazel_slsa3.yml @@ -93,7 +93,7 @@ on: The name of the folder containing the built artifacts. There is a random hash at the beginning of it in form -binaries to avoid collisions. value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).artifacts-download-name }} - + artifacts-download-sha256: description: "SHA256 of the uploaded tarball of built artifacts." value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).artifacts-download-sha256 }} From cd75049e37e62f1acdcff4433d1385d2e36fd749 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 07:58:20 +0000 Subject: [PATCH 25/52] Preprend UNTRUSTED, GITHUB_ENV --> GITHUB_OUTPUT & no need for caps, docker_image --> env_image Signed-off-by: Noah Elzner --- .github/workflows/builder_bazel_slsa3.yml | 4 +-- internal/builders/bazel/action.yml | 42 +++++++++++------------ internal/builders/bazel/build.sh | 26 +++++++------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/builder_bazel_slsa3.yml b/.github/workflows/builder_bazel_slsa3.yml index 4a607a5648..4c3e15ac76 100644 --- a/.github/workflows/builder_bazel_slsa3.yml +++ b/.github/workflows/builder_bazel_slsa3.yml @@ -35,8 +35,8 @@ on: required: false type: string default: "" - docker-image: - description: "Docker Image for build environment to run on" + env-image: + description: "Image for build environment to run on" required: false type: string default: "" diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index af4a445780..aa65f790de 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -68,40 +68,40 @@ runs: distribution: "${{ fromJson(inputs.slsa-workflow-inputs).user-java-distribution }}" java-version: "${{ fromJson(inputs.slsa-workflow-inputs).user-java-version }}" - - name: Check for Docker Image - id: docker + - name: Check for Environment Image + id: env-image shell: bash run: | - if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).docker-image }}" ]] + if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).env-image }}" ]] then - echo "No Docker Image provided. Will build without." - echo "USE_DOCKER=false" >> $GITHUB_ENV + echo "No Environment Image provided. Will build without." + echo "use_image=false" >> "$GITHUB_OUTPUT" else - echo "Docker image provided. Running build on Docker Image." - echo "USE_DOCKER=true" >> $GITHUB_ENV + echo "Environment image provided. Running build on Environment Image." + echo "use_image=true" >> "$GITHUB_OUTPUT" fi - - name: Build on Docker Image - if: env.USE_DOCKER == 'true' + - name: Build on Environment Image + if: ${{ steps.env-image.outputs.use_image }} == 'true' env: - TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} - FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} - NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} - INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} - DOCKER_IMAGE: ${{ fromJson(inputs.slsa-workflow-inputs).docker-image }} + UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} + UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} + UNTRUSTED_NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} + UNTRUSTED_INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} + UNTRUSTED_ENV_IMAGE: ${{ fromJson(inputs.slsa-workflow-inputs).env-image }} shell: bash run: | - docker pull $DOCKER_IMAGE + docker pull $UNTRUSTED_ENV_IMAGE curr_dir=$(basename "$(pwd)") - docker run --rm --env TARGETS=${TARGETS} --env FLAGS=${FLAGS} --env NEEDS_RUNFILES=${NEEDS_RUNFILES} --env INCLUDES_JAVA=${INCLUDES_JAVA} -v $PWD/../:/src -w /src $DOCKER_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" + docker run --rm --env TARGETS=${UNTRUSTED_TARGETS} --env FLAGS=${UNTRUSTED_FLAGS} --env NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build - if: env.USE_DOCKER == 'false' + if: ${{ steps.env-image.outputs.use_image }} == 'false' env: - TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} - FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} - NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} - INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} + UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} + UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} + UNTRUSTED_NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} + UNTRUSTED_INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} shell: bash run: ./../__TOOL_ACTION_DIR__/build.sh diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index 5c411a3578..447749498f 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -22,12 +22,12 @@ binaries_dir="bazel_builder_binaries_to_upload_to_gh_7bc972367cb286b7f36ab4457f0 mkdir ${binaries_dir} # Transfer flags and targets to their respective arrays -IFS=' ' read -r -a build_flags <<< "${FLAGS}" -IFS=' ' read -r -a build_targets <<< "${TARGETS}" +IFS=' ' read -r -a build_flags <<< "${UNTRUSTED_FLAGS}" +IFS=' ' read -r -a build_targets <<< "${UNTRUSTED_TARGETS}" # If the targets includes Java targets, include Java build flag # and add Github Runner Java rule to WORKSPACE -if [[ "${INCLUDES_JAVA}" == "true" ]] +if [[ "${UNTRUSTED_INCLUDES_JAVA}" == "true" ]] then build_flags+=("--java_runtime_version=myjdk") @@ -52,7 +52,7 @@ declare -A targets_set ################################################ for input in "${build_targets[@]}"; do - + # Using bazel query extracts all targets from a glob pattern. # Thus we can change Java targets to their _deploy.jar target. for target in $(bazel query "$input"); do @@ -60,12 +60,12 @@ for input in "${build_targets[@]}"; do # Check to see if the target is a Java target. If it is the output is a Java target. # Note: targets that already have the _deploy.jar suffix will have no output from the query output=$(bazel query "kind(java_binary, $target)" 2>/dev/null) - + # If there is a Java target without _deploy.jar suffix, add suffix, build and add to target set. if [[ -n "$output" ]] then bazel build "${build_flags[@]}" "${target}_deploy.jar" - targets_set["${target}_deploy.jar"]="1" + targets_set["${target}_deploy.jar"]="1" else # Build target regularly. bazel build "${build_flags[@]}" "$target" @@ -81,7 +81,7 @@ done ################################################ for curr_target in "${!targets_set[@]}"; do - + # Removes everything up to and including the first colon # "//src/internal:fib" --> "fib" binary_name=${curr_target#*:} @@ -107,7 +107,7 @@ for curr_target in "${!targets_set[@]}"; do # Copy JAR to artifact-specific dir in ./${binaries_dir} and remove symbolic links. file="$bazel_generated" cp -Lr "$file" "./${binaries_dir}/$run_script_name" - + # Get the path the to run-script associated with the {$curr_target}_deploy.jar # If the user inputted the path to their local JAVABIN insert that into the run-script to define it. # Inputting a local path to JAVABIN is needed or else run-script will not work as it points to Github Runner JAVABIN @@ -118,11 +118,11 @@ for curr_target in "${!targets_set[@]}"; do # to run the run-script themselves, which would not be possible as it is either set to the Github Runner VM Java bin path # if no flag to USER_LOCAL_JAVABIN is passed in their workflow or to the path passed in their flag. awk -v n=66 -v s=' --local_javabin=*) USER_JAVA_BIN=( "${1#--local_javabin=}" ) ;;' 'NR == n {print s} {print}' "$run_script_path" > temp_file && mv -f temp_file "$run_script_path" - + # Updates Java Bin in run-script after the flags get proccessed awk -v n=127 -v s='' 'NR == n {print s} {print}' "$run_script_path" > temp_file && mv -f temp_file "$run_script_path" awk -v n=128 -v s='if [[ -n $USER_JAVA_BIN ]]; then JAVABIN=$USER_JAVA_BIN; fi' 'NR == n {print s} {print}' "$run_script_path" > temp_file && mv -f temp_file "$run_script_path" - + cp -L "$run_script_path" "./${binaries_dir}/$run_script_name" ################################################ @@ -130,7 +130,7 @@ for curr_target in "${!targets_set[@]}"; do # Logic for Non-Java Targets # # # ################################################ - + else ################################################ @@ -139,7 +139,7 @@ for curr_target in "${!targets_set[@]}"; do # # ################################################ - if [[ "${NEEDS_RUNFILES}" == "true" ]] + if [[ "${UNTRUSTED_NEEDS_RUNFILES}" == "true" ]] then # Get file(s) generated from build with respect to the target bazel_generated=$(bazel cquery --output=starlark --starlark:expr="'\n'.join([f.path for f in target.files.to_list()])" "$curr_target" 2>/dev/null) @@ -178,7 +178,7 @@ for curr_target in "${!targets_set[@]}"; do else # Get file(s) generated from build with respect to the target bazel_generated=$(bazel cquery --output=starlark --starlark:expr="'\n'.join([f.path for f in target.files.to_list()])" "$curr_target" 2>/dev/null) - + # Uses a Starlark expression to pass new line seperated list of file(s) into the set of files while read -r file; do cp -L "$file" ./${binaries_dir} From dd40f91dd7477c0bc301bb249c0198eb60f3217b Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 08:08:37 +0000 Subject: [PATCH 26/52] UNTRUSTED_ prepend Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 2 +- internal/builders/bazel/rebuilder.sh | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index aa65f790de..e8c512cd82 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -93,7 +93,7 @@ runs: run: | docker pull $UNTRUSTED_ENV_IMAGE curr_dir=$(basename "$(pwd)") - docker run --rm --env TARGETS=${UNTRUSTED_TARGETS} --env FLAGS=${UNTRUSTED_FLAGS} --env NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" + docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build if: ${{ steps.env-image.outputs.use_image }} == 'false' diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index de9a043cff..dd8def48cd 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -261,13 +261,13 @@ fi # The name map will convert and export the key strings of inputs to # match with the environment variables of the Bazel Builder build.sh declare -A name_mapping -name_mapping["targets"]="TARGETS" -name_mapping["flags"]="FLAGS" -name_mapping["docker-image"]="DOCKER_IMAGE" +name_mapping["targets"]="UNTRUSTED_TARGETS" +name_mapping["flags"]="UNTRUSTED_FLAGS" +name_mapping["docker-image"]="UNTRUSTED_DOCKER_IMAGE" # Note: These boolean inputs are now dealed with as strings -name_mapping["includes-java"]="INCLUDES_JAVA" -name_mapping["needs-runfiles"]="NEEDS_RUNFILES" +name_mapping["includes-java"]="UNTRUSTED_INCLUDES_JAVA" +name_mapping["needs-runfiles"]="UNTRUSTED_NEEDS_RUNFILES" # Export the inputs for later use for key in "${!data[@]}"; do @@ -334,16 +334,16 @@ echo -e "${CYAN}======================================================${RESET}" # Conditionals for docker images depend on if a Docker Image was use to build on Github. # If a Docker Image was not used to build on Github, then build locally. This is done to # ensure consistent build environment between both platforms. -if [[ -n ${DOCKER_IMAGE:-} ]]; then +if [[ -n ${UNTRUSTED_DOCKER_IMAGE:-} ]]; then cd - - sudo docker pull "$DOCKER_IMAGE" + sudo docker pull "$UNTRUSTED_DOCKER_IMAGE" echo "" echo -e "${CYAN}======================================================${RESET}" type_writer "๐Ÿ”จ---> Rebuilding with Docker Image Environment..." # Mount docker image on this directory as workdir to gain access to script env echo -e "${CYAN}======================================================${RESET}" echo "" - sudo docker run --env repo_name="$repo_name" --env TARGETS="${TARGETS}" --env FLAGS="${FLAGS}" --env NEEDS_RUNFILES="${NEEDS_RUNFILES}" --env INCLUDES_JAVA="${INCLUDES_JAVA}" --rm -v "$PWD":/workdir -w /workdir "$DOCKER_IMAGE" /bin/sh -c "cd $repo_name && ./../build.sh" + sudo docker run --env repo_name="$repo_name" --env UNTRUSTED_TARGETS="${UNTRUSTED_TARGETS}" --env UNTRUSTED_FLAGS="${UNTRUSTED_FLAGS}" --env UNTRUSTED_NEEDS_RUNFILES="${UNTRUSTED_NEEDS_RUNFILES}" --env UNTRUSTED_INCLUDES_JAVA="${UNTRUSTED_INCLUDES_JAVA}" --rm -v "$PWD":/workdir -w /workdir "$UNTRUSTED_DOCKER_IMAGE" /bin/sh -c "cd $repo_name && ./../build.sh" echo "" echo -e "${CYAN}======================================================${RESET}" echo -e "${CYAN}|${RESET}${YELLOW}${UNDERLINE} โœ… Artifacts Rebuilt! โœ… ${RESET}${CYAN}|${RESET}" @@ -375,7 +375,7 @@ fi # If Docker Image was used to build on Github, we need to cd into repo # to access the binaries directory. -if [[ -n ${DOCKER_IMAGE:-} ]]; then +if [[ -n ${UNTRUSTED_DOCKER_IMAGE:-} ]]; then cd "$repo_name" fi From b18ec2717f4a1a99737cb3930b796d27d84abd44 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 08:12:28 +0000 Subject: [PATCH 27/52] prepend Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index dd8def48cd..d0a08dc11d 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -304,7 +304,7 @@ cd "$repo_name" # Check to see if JAVA_HOME is set then empty to # avoid triggering unbound variable error. -if [[ "${INCLUDES_JAVA}" == "true" ]] +if [[ "${UNTRUSTED_INCLUDES_JAVA}" == "true" ]] then if [[ ! -v JAVA_HOME || -z "${JAVA_HOME}" ]]; then # if JAVA_HOME is empty, set to jdk bin path from $(which java) From 0c6aaa3642b0b173a9b2d84506123cf1cd281569 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 08:28:14 +0000 Subject: [PATCH 28/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index 447749498f..dc0c14d377 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -74,6 +74,10 @@ for input in "${build_targets[@]}"; do done done +ls +tree +echo $(pwd) + ################################################ # # # Copy Needed Artifacts To Binaries Dir # From e34d9ec4bbf396365eb4fdb762d708b5a735a38e Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 08:29:17 +0000 Subject: [PATCH 29/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index dc0c14d377..9ad22b5638 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -101,7 +101,7 @@ for curr_target in "${!targets_set[@]}"; do then # Uses _deploy.jar as a field seperator and grabs the field before it. run_script_name=$(echo "$binary_name" | awk -F'_deploy.jar' '{print $1}') - + echo $run_script_name # Create dir for artifact and its runfiles mkdir "./${binaries_dir}/$run_script_name" From 20ec13516243199a6435c7b1353b3ccf22bb4e0b Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 08:38:48 +0000 Subject: [PATCH 30/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index 9ad22b5638..8657224190 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -103,7 +103,7 @@ for curr_target in "${!targets_set[@]}"; do run_script_name=$(echo "$binary_name" | awk -F'_deploy.jar' '{print $1}') echo $run_script_name # Create dir for artifact and its runfiles - mkdir "./${binaries_dir}/$run_script_name" + mkdir -p "./${binaries_dir}/$run_script_name" # Get the absolute path to output of Java JAR artifact. bazel_generated=$(bazel cquery --output=starlark --starlark:expr="'\n'.join([f.path for f in target.files.to_list()])" "$curr_target" 2>/dev/null) From f6349e0978e19c995824aa13af0b8bd68eda31d8 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 08:39:41 +0000 Subject: [PATCH 31/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index 8657224190..863beb6f1e 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -103,7 +103,7 @@ for curr_target in "${!targets_set[@]}"; do run_script_name=$(echo "$binary_name" | awk -F'_deploy.jar' '{print $1}') echo $run_script_name # Create dir for artifact and its runfiles - mkdir -p "./${binaries_dir}/$run_script_name" + mkdir -p "./${binaries_dir}/${run_script_name}" # Get the absolute path to output of Java JAR artifact. bazel_generated=$(bazel cquery --output=starlark --starlark:expr="'\n'.join([f.path for f in target.files.to_list()])" "$curr_target" 2>/dev/null) From 543d63209a1c5307498f4d9c37a382daec12dab1 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 08:43:03 +0000 Subject: [PATCH 32/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index 863beb6f1e..b5570cac54 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -102,6 +102,7 @@ for curr_target in "${!targets_set[@]}"; do # Uses _deploy.jar as a field seperator and grabs the field before it. run_script_name=$(echo "$binary_name" | awk -F'_deploy.jar' '{print $1}') echo $run_script_name + tree # Create dir for artifact and its runfiles mkdir -p "./${binaries_dir}/${run_script_name}" From d3e22491aa6a27e37119d7cc612b9a313fd4d9be Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 09:03:58 +0000 Subject: [PATCH 33/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index b5570cac54..08bc9e0641 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -103,8 +103,13 @@ for curr_target in "${!targets_set[@]}"; do run_script_name=$(echo "$binary_name" | awk -F'_deploy.jar' '{print $1}') echo $run_script_name tree + + # debug fix attempt + cd $binaries_dir + mkdir "$run_script_name" + cd - # Create dir for artifact and its runfiles - mkdir -p "./${binaries_dir}/${run_script_name}" + #mkdir -p "./${binaries_dir}/${run_script_name}" # Get the absolute path to output of Java JAR artifact. bazel_generated=$(bazel cquery --output=starlark --starlark:expr="'\n'.join([f.path for f in target.files.to_list()])" "$curr_target" 2>/dev/null) From 3abc64a94061c44260c177bd7d59b5d4c38ced5e Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 09:08:50 +0000 Subject: [PATCH 34/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index 08bc9e0641..ca07430e97 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -103,10 +103,12 @@ for curr_target in "${!targets_set[@]}"; do run_script_name=$(echo "$binary_name" | awk -F'_deploy.jar' '{print $1}') echo $run_script_name tree - + ls # debug fix attempt cd $binaries_dir + ls mkdir "$run_script_name" + ls cd - # Create dir for artifact and its runfiles #mkdir -p "./${binaries_dir}/${run_script_name}" From 529f50f391c8765b56f0a0862cfd2fde960053f5 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Fri, 11 Aug 2023 02:28:20 -0700 Subject: [PATCH 35/52] Update action.yml Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index e8c512cd82..fdba46f192 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -82,7 +82,7 @@ runs: fi - name: Build on Environment Image - if: ${{ steps.env-image.outputs.use_image }} == 'true' + if: ${{ steps.env-image.outputs.use_image }} == true env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} @@ -96,7 +96,7 @@ runs: docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build - if: ${{ steps.env-image.outputs.use_image }} == 'false' + if: ${{ steps.env-image.outputs.use_image }} == false env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From 51d14a3f1bd572309c09eaebfc3a792f795819a9 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 09:34:27 +0000 Subject: [PATCH 36/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index e8c512cd82..50f3d9c6ff 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -75,14 +75,14 @@ runs: if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).env-image }}" ]] then echo "No Environment Image provided. Will build without." - echo "use_image=false" >> "$GITHUB_OUTPUT" + echo "USE_DOCKER=false" >> $GITHUB_ENV else - echo "Environment image provided. Running build on Environment Image." - echo "use_image=true" >> "$GITHUB_OUTPUT" + echo "Docker image provided. Running build on Docker Image." + echo "USE_DOCKER=true" >> $GITHUB_ENV fi - name: Build on Environment Image - if: ${{ steps.env-image.outputs.use_image }} == 'true' + if: env.USE_DOCKER == 'true' env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} @@ -96,7 +96,7 @@ runs: docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build - if: ${{ steps.env-image.outputs.use_image }} == 'false' + if: env.USE_DOCKER == 'false' env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From 3fdb6ba0bbe62fb9d7eb1005b63019484b802762 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 09:44:23 +0000 Subject: [PATCH 37/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 50f3d9c6ff..9386333c78 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -75,14 +75,15 @@ runs: if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).env-image }}" ]] then echo "No Environment Image provided. Will build without." - echo "USE_DOCKER=false" >> $GITHUB_ENV + echo "::set-output name=boolean_output::${{ toJSON($i) }}" + else echo "Docker image provided. Running build on Docker Image." - echo "USE_DOCKER=true" >> $GITHUB_ENV + # echo "USE_DOCKER=true" >> $GITHUB_ENV fi - name: Build on Environment Image - if: env.USE_DOCKER == 'true' + if: fromJSON(steps.output_producer.outputs.boolean_output) env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From 89de881bb6eace9cc4a57015124627ba42e590ef Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 10:00:50 +0000 Subject: [PATCH 38/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 9386333c78..6960dcca62 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -75,12 +75,13 @@ runs: if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).env-image }}" ]] then echo "No Environment Image provided. Will build without." - echo "::set-output name=boolean_output::${{ toJSON($i) }}" - + i=false else echo "Docker image provided. Running build on Docker Image." # echo "USE_DOCKER=true" >> $GITHUB_ENV + i=true fi + echo "::set-output name=boolean_output::${{ toJSON($i) }}" - name: Build on Environment Image if: fromJSON(steps.output_producer.outputs.boolean_output) @@ -97,7 +98,7 @@ runs: docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build - if: env.USE_DOCKER == 'false' + if: false == fromJSON(steps.output_producer.outputs.boolean_output) env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From 3e2d2915f2bbc90424917ab49f9d2e6ac92dfd32 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 10:14:46 +0000 Subject: [PATCH 39/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 6960dcca62..4430f420f7 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -75,16 +75,15 @@ runs: if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).env-image }}" ]] then echo "No Environment Image provided. Will build without." - i=false + echo "use_env_image=false >> $GITHUB_OUTPUT" else echo "Docker image provided. Running build on Docker Image." - # echo "USE_DOCKER=true" >> $GITHUB_ENV + # echo "use_env_image=true >> $GITHUB_OUTPUT"" i=true fi - echo "::set-output name=boolean_output::${{ toJSON($i) }}" - name: Build on Environment Image - if: fromJSON(steps.output_producer.outputs.boolean_output) + if: steps.env-image.outputs.use_env_image env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} @@ -98,7 +97,7 @@ runs: docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build - if: false == fromJSON(steps.output_producer.outputs.boolean_output) + if: steps.env-image.outputs.use_env_image env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From a70bcc783c4e3c874931c938434c1ed47caa202e Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 10:19:08 +0000 Subject: [PATCH 40/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 4430f420f7..66870d9a30 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -90,8 +90,11 @@ runs: UNTRUSTED_NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} UNTRUSTED_INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} UNTRUSTED_ENV_IMAGE: ${{ fromJson(inputs.slsa-workflow-inputs).env-image }} + test: ${{ steps.env-image.outputs.use_env_image }} shell: bash run: | + echo "docker block:" + echo $test docker pull $UNTRUSTED_ENV_IMAGE curr_dir=$(basename "$(pwd)") docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" @@ -103,8 +106,12 @@ runs: UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} UNTRUSTED_NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} UNTRUSTED_INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} + test: ${{ steps.env-image.outputs.use_env_image }} shell: bash - run: ./../__TOOL_ACTION_DIR__/build.sh + run: | + echo "running" + echo $test + ./../__TOOL_ACTION_DIR__/build.sh # rng generates a random number to avoid name collision in artifacts # when multiple workflows run concurrently. From e69e6cbc7928277b4d26c6ba503a1249bb51d667 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 10:32:29 +0000 Subject: [PATCH 41/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 66870d9a30..adf7a250d4 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -78,12 +78,12 @@ runs: echo "use_env_image=false >> $GITHUB_OUTPUT" else echo "Docker image provided. Running build on Docker Image." - # echo "use_env_image=true >> $GITHUB_OUTPUT"" + # echo "use_env_image=true >> $GITHUB_OUTPUT" i=true fi - name: Build on Environment Image - if: steps.env-image.outputs.use_env_image + if: steps.env-image.outputs.use_env_image == true env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} @@ -100,7 +100,7 @@ runs: docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build - if: steps.env-image.outputs.use_env_image + if: steps.env-image.outputs.use_env_image == false env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From 98940837be62d47e0269401c4b5e6a27b450ebbf Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 10:45:48 +0000 Subject: [PATCH 42/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index adf7a250d4..d3ad6c8e89 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -78,12 +78,11 @@ runs: echo "use_env_image=false >> $GITHUB_OUTPUT" else echo "Docker image provided. Running build on Docker Image." - # echo "use_env_image=true >> $GITHUB_OUTPUT" - i=true + echo "use_env_image=true >> $GITHUB_OUTPUT" fi - name: Build on Environment Image - if: steps.env-image.outputs.use_env_image == true + if: ${{ steps.env-image.outputs.use_env_image == 'true' }} env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} @@ -100,7 +99,7 @@ runs: docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" - id: build - if: steps.env-image.outputs.use_env_image == false + if: ${{ steps.env-image.outputs.use_env_image == 'false' }} env: UNTRUSTED_TARGETS: ${{ fromJson(inputs.slsa-workflow-inputs).targets }} UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} From 78a55c09fb98969584829c8286ce564fb0992661 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 11:02:06 +0000 Subject: [PATCH 43/52] debug Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index d3ad6c8e89..717f0fe263 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -75,10 +75,10 @@ runs: if [[ -z "${{ fromJson(inputs.slsa-workflow-inputs).env-image }}" ]] then echo "No Environment Image provided. Will build without." - echo "use_env_image=false >> $GITHUB_OUTPUT" + echo "use_env_image=false" >> $GITHUB_OUTPUT else echo "Docker image provided. Running build on Docker Image." - echo "use_env_image=true >> $GITHUB_OUTPUT" + echo "use_env_image=true" >> $GITHUB_OUTPUT fi - name: Build on Environment Image From 15bba705cf96bc841b818bbafe7f968b1cd5bc00 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Fri, 11 Aug 2023 04:10:06 -0700 Subject: [PATCH 44/52] remove debug Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/build.sh | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/internal/builders/bazel/build.sh b/internal/builders/bazel/build.sh index ca07430e97..447749498f 100755 --- a/internal/builders/bazel/build.sh +++ b/internal/builders/bazel/build.sh @@ -74,10 +74,6 @@ for input in "${build_targets[@]}"; do done done -ls -tree -echo $(pwd) - ################################################ # # # Copy Needed Artifacts To Binaries Dir # @@ -101,17 +97,9 @@ for curr_target in "${!targets_set[@]}"; do then # Uses _deploy.jar as a field seperator and grabs the field before it. run_script_name=$(echo "$binary_name" | awk -F'_deploy.jar' '{print $1}') - echo $run_script_name - tree - ls - # debug fix attempt - cd $binaries_dir - ls - mkdir "$run_script_name" - ls - cd - + # Create dir for artifact and its runfiles - #mkdir -p "./${binaries_dir}/${run_script_name}" + mkdir "./${binaries_dir}/$run_script_name" # Get the absolute path to output of Java JAR artifact. bazel_generated=$(bazel cquery --output=starlark --starlark:expr="'\n'.join([f.path for f in target.files.to_list()])" "$curr_target" 2>/dev/null) From 8ba63f6c89de5a7c1bafd6f3faaf11e033debdfe Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 16:49:15 +0000 Subject: [PATCH 45/52] set -euo pipefail Signed-off-by: Noah Elzner --- internal/builders/bazel/action.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 717f0fe263..1f2fb828c1 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -89,11 +89,9 @@ runs: UNTRUSTED_NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} UNTRUSTED_INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} UNTRUSTED_ENV_IMAGE: ${{ fromJson(inputs.slsa-workflow-inputs).env-image }} - test: ${{ steps.env-image.outputs.use_env_image }} shell: bash run: | - echo "docker block:" - echo $test + set -euo pipefail docker pull $UNTRUSTED_ENV_IMAGE curr_dir=$(basename "$(pwd)") docker run --rm --env UNTRUSTED_TARGETS=${UNTRUSTED_TARGETS} --env UNTRUSTED_FLAGS=${UNTRUSTED_FLAGS} --env UNTRUSTED_NEEDS_RUNFILES=${UNTRUSTED_NEEDS_RUNFILES} --env UNTRUSTED_INCLUDES_JAVA=${UNTRUSTED_INCLUDES_JAVA} -v $PWD/../:/src -w /src $UNTRUSTED_ENV_IMAGE /bin/sh -c "ls && tree && cd $curr_dir && ls && tree && ./../__TOOL_ACTION_DIR__/build.sh" @@ -105,11 +103,8 @@ runs: UNTRUSTED_FLAGS: ${{ fromJson(inputs.slsa-workflow-inputs).flags }} UNTRUSTED_NEEDS_RUNFILES: ${{ fromJson(inputs.slsa-workflow-inputs).needs-runfiles }} UNTRUSTED_INCLUDES_JAVA: ${{ fromJson(inputs.slsa-workflow-inputs).includes-java }} - test: ${{ steps.env-image.outputs.use_env_image }} shell: bash run: | - echo "running" - echo $test ./../__TOOL_ACTION_DIR__/build.sh # rng generates a random number to avoid name collision in artifacts From fd11a629315e935e28650c975b685b70091b4656 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 16:50:54 +0000 Subject: [PATCH 46/52] lowercase typespeed Signed-off-by: Noah Elzner --- internal/builders/bazel/rebuilder.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index d0a08dc11d..8e2f27f82e 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -46,13 +46,13 @@ UNDERLINE="\033[4m" # # ################################################ -TYPE_SPEED=0.02 +type_speed=0.02 function type_writer { text="$1" for (( i=0; i<${#text}; i++ )); do echo -n "${text:$i:1}" - sleep $TYPE_SPEED + sleep $type_speed done echo "" } From ce9dc3bcc22fbceb5935b6edf22a474e000a0cd9 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 16:56:20 +0000 Subject: [PATCH 47/52] add digest with todo Signed-off-by: Noah Elzner --- .github/workflows/builder_bazel_slsa3.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/builder_bazel_slsa3.yml b/.github/workflows/builder_bazel_slsa3.yml index 4c3e15ac76..3b541ddb44 100644 --- a/.github/workflows/builder_bazel_slsa3.yml +++ b/.github/workflows/builder_bazel_slsa3.yml @@ -40,6 +40,13 @@ on: required: false type: string default: "" + env-image-digest: + description: > + TODO(#2630): Add verification method for digest. + The image digest of the environment image. + This must be specified in order to verify the image. + required: false + type: string needs-runfiles: description: > A boolean input that if true will package the artifact's runfiles along with the artifact. From 113e7340170d0e1201876d0bd5f4d8ca3cbba784 Mon Sep 17 00:00:00 2001 From: Noah Elzner <78953604+enteraga6@users.noreply.github.com> Date: Fri, 11 Aug 2023 12:31:00 -0700 Subject: [PATCH 48/52] start rebuilder doc Signed-off-by: Noah Elzner <78953604+enteraga6@users.noreply.github.com> --- internal/builders/bazel/README.md | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/internal/builders/bazel/README.md b/internal/builders/bazel/README.md index 7e844d6f62..0536cb49e3 100644 --- a/internal/builders/bazel/README.md +++ b/internal/builders/bazel/README.md @@ -27,6 +27,9 @@ workflow the "Bazel builder" from now on. - [Workflow Outputs](#workflow-outputs) - [Provenance Format](#provenance-format) - [Provenance Example](#provenance-example) +- [Verification and Rebuilding](#verification-and-rebuilding) + - [Verification](#verification) + - [Rebuilding](#rebuilding) @@ -319,3 +322,37 @@ The following is an example of the generated provenance. } } ``` + +## Verification and Rebuilding + +### Verification + +Verification of the provenance generated for an artifact can be done with one of two ways. Using the [`slsa-verifier`](https://github.com/slsa-framework/slsa-verifier) +as follows on their [instructions](https://github.com/slsa-framework/slsa-verifier#verification-for-github-builders) for verification of Github builder artifacts. + +Verification can also be done through the passing the `--verify` flag to the rebuilder. + +### Rebuilding +The Bazel Rebuilder takes in the following arguments on the command line + +Arguments: + +| Argument Name to Rebuilder | Required For Rebuilder | Additionally Required for Verification | Description | +| ------------------------- | -------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--artifact_path=` | Yes | | Path to the artifact to rebuild and compares checksums with. | +| `--prov_path=` | Yes | | Path to the provenance of the artifact that is being rebuilt. | +| `--source_uri=` | Yes | | expected source repository that should have produced the binary, e.g. github.com/some/repo +| `--builder_id=` | No | Yes | The unique builder ID who created the provenance +| `--docker_image=` | No | No | A published image to be pull to build on top of +| `--verify` | No | Yes | Flag to verify provenance for artifact being rebuilt +| `--verbose` | No | No | Flag to include extra output to track progress +| `--cleanup` | No | No | Removes cloned repos (`source_uri` and `slsa-verifier`) as well as directory for rebuilt artifacts + +The rebuilder does the following steps: verifies the provenance for the artifact to rebuild, +parses out the attested build process from the provenance, +clones the repo that produced it, +rebuilds the inputted artifact with the attest build process, +and then compares checksums + +An example usage of the rebuilder is the following command: +`./rebuilder.sh --artifact_path== --prov_path= --source_uri= --builder_id= --docker_image= --verify --verbose` From 8eb66adfacb49ce716c000caa15eeb1eab6d7033 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 19:57:50 +0000 Subject: [PATCH 49/52] docker --> env && complete rebuilder docs Signed-off-by: Noah Elzner --- internal/builders/bazel/README.md | 37 ++++++++++++++++++---------- internal/builders/bazel/rebuilder.sh | 26 +++++++++---------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/internal/builders/bazel/README.md b/internal/builders/bazel/README.md index 0536cb49e3..346dad07ea 100644 --- a/internal/builders/bazel/README.md +++ b/internal/builders/bazel/README.md @@ -327,13 +327,14 @@ The following is an example of the generated provenance. ### Verification -Verification of the provenance generated for an artifact can be done with one of two ways. Using the [`slsa-verifier`](https://github.com/slsa-framework/slsa-verifier) -as follows on their [instructions](https://github.com/slsa-framework/slsa-verifier#verification-for-github-builders) for verification of Github builder artifacts. +Verification of the provenance generated for an artifact can be done with one of two ways. Using the [`slsa-verifier`](https://github.com/slsa-framework/slsa-verifier) +as follows on their [instructions](https://github.com/slsa-framework/slsa-verifier#verification-for-github-builders) for verification of Github builder artifacts. Verification can also be done through the passing the `--verify` flag to the rebuilder. ### Rebuilding -The Bazel Rebuilder takes in the following arguments on the command line +To rebuild your artifacts and check for reproducible builds use the Bazel Rebuilder, +which takes in the following arguments on the command line. Arguments: @@ -343,16 +344,26 @@ Arguments: | `--prov_path=` | Yes | | Path to the provenance of the artifact that is being rebuilt. | | `--source_uri=` | Yes | | expected source repository that should have produced the binary, e.g. github.com/some/repo | `--builder_id=` | No | Yes | The unique builder ID who created the provenance -| `--docker_image=` | No | No | A published image to be pull to build on top of +| `--env_image=` | No | | A published image to be pull to build on top of | `--verify` | No | Yes | Flag to verify provenance for artifact being rebuilt -| `--verbose` | No | No | Flag to include extra output to track progress -| `--cleanup` | No | No | Removes cloned repos (`source_uri` and `slsa-verifier`) as well as directory for rebuilt artifacts - -The rebuilder does the following steps: verifies the provenance for the artifact to rebuild, -parses out the attested build process from the provenance, -clones the repo that produced it, -rebuilds the inputted artifact with the attest build process, -and then compares checksums +| `--verbose` | No | | Flag to include extra output to track progress +| `--cleanup` | No | | Removes cloned repos (`source_uri` and `slsa-verifier`) as well as directory for rebuilt artifacts + +The rebuilder does the following: +~~~ +1. Verifies the provenance for the artifact to rebuild, +2. Parses out the attested build process from the provenance, +3. Clones the repo that produced it, +4. Rebuilds the inputted artifact with the attest build process, +5. Compares checksums for reproducibility +~~~ An example usage of the rebuilder is the following command: -`./rebuilder.sh --artifact_path== --prov_path= --source_uri= --builder_id= --docker_image= --verify --verbose` +`./rebuilder.sh --artifact_path== --prov_path= --source_uri= --builder_id= --env_image= --verify --verbose` + +Using an image that dictates the build environment is needed to make reproducible rebuilding possible. The artifact +must be built on an image from a registry originally during the Github Actions Workflow run to also build on a provided image during the rebuilding process. + +If no image was provided or if the artifact was not originally built on an image, the rebuilding process will use +the user's local environment, thus diminishing the chances of a reproducible rebuild due to non-determinism from the +different build environments. \ No newline at end of file diff --git a/internal/builders/bazel/rebuilder.sh b/internal/builders/bazel/rebuilder.sh index 8e2f27f82e..29cca77d30 100755 --- a/internal/builders/bazel/rebuilder.sh +++ b/internal/builders/bazel/rebuilder.sh @@ -82,14 +82,14 @@ cleanup=0 function usage() { if [[ $verify ]]; then echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" - echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET}= ${YELLOW}--prov_path${RESET}= ${YELLOW}--source_uri${RESET}= ${YELLOW}--builder_id${RESET}= ${MAGENTA}[--env_image]${RESET}= ${MAGENTA}[--verify]${RESET}" echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" - echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET}= ${YELLOW}--prov_path${RESET}= ${MAGENTA}[--env_image]${RESET}=" else echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to ONLY rebuild the artifact:${RESET}" - echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${MAGENTA}[--docker_image]${RESET} " + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET}= ${YELLOW}--prov_path${RESET}= ${MAGENTA}[--env_image]${RESET}=" echo -e "${RED}[ERROR] ${LIGHT_RED}Wrong usage. Usage to verify AND rebuild artifact:${RESET}" - echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET} ${YELLOW}--prov_path${RESET} ${YELLOW}--source_uri${RESET} ${YELLOW}--builder_id${RESET} ${MAGENTA}[--docker_image]${RESET} ${MAGENTA}[--verify]${RESET}" + echo -e "${CYAN}Usage: $0 ${YELLOW}--artifact_path${RESET}= ${YELLOW}--prov_path${RESET}= ${YELLOW}--source_uri${RESET}= ${YELLOW}--builder_id${RESET}= ${MAGENTA}[--env_image]${RESET}= ${MAGENTA}[--verify]${RESET}" fi } @@ -101,7 +101,7 @@ function process_argument() { --prov_path=*) prov_path="${1#--prov_path=}" ;; --source_uri=*) source_uri="${1#--source_uri=}" ;; --builder_id=*) builder_id="${1#--builder_id=}" ;; - --docker_image=*) docker_image="${1#--docker_image=}" ;; + --env_image=*) env_image="${1#--env_image=}" ;; --verify) verify=1 ;; --verbose) verbose=1 ;; --cleanup) cleanup=1 ;; @@ -183,8 +183,8 @@ if [[ $verbose -eq 1 ]]; then echo -e "${CYAN}builder_id: ${GREEN}$builder_id${RESET}" fi - if [[ -n ${docker_image:-} ]]; then - echo -e "${CYAN}docker_image: ${GREEN}$docker_image${RESET}" + if [[ -n ${env_image:-} ]]; then + echo -e "${CYAN}env_image: ${GREEN}$env_image${RESET}" fi echo -e "${CYAN}verify: ${GREEN}$verify${RESET}" @@ -263,7 +263,7 @@ fi declare -A name_mapping name_mapping["targets"]="UNTRUSTED_TARGETS" name_mapping["flags"]="UNTRUSTED_FLAGS" -name_mapping["docker-image"]="UNTRUSTED_DOCKER_IMAGE" +name_mapping["env-image"]="UNTRUSTED_DOCKER_IMAGE" # Note: These boolean inputs are now dealed with as strings name_mapping["includes-java"]="UNTRUSTED_INCLUDES_JAVA" @@ -331,7 +331,7 @@ echo -e "${CYAN}======================================================${RESET}" echo -e "${CYAN}|${RESET}${YELLOW}${UNDERLINE} ๐Ÿ”จ Starting the Rebuild Process ๐Ÿ”จ ${RESET}${CYAN}|${RESET}" echo -e "${CYAN}======================================================${RESET}" -# Conditionals for docker images depend on if a Docker Image was use to build on Github. +# Conditionals for environment images depend on if a Docker Image was use to build on Github. # If a Docker Image was not used to build on Github, then build locally. This is done to # ensure consistent build environment between both platforms. if [[ -n ${UNTRUSTED_DOCKER_IMAGE:-} ]]; then @@ -339,7 +339,7 @@ if [[ -n ${UNTRUSTED_DOCKER_IMAGE:-} ]]; then sudo docker pull "$UNTRUSTED_DOCKER_IMAGE" echo "" echo -e "${CYAN}======================================================${RESET}" - type_writer "๐Ÿ”จ---> Rebuilding with Docker Image Environment..." # Mount docker image on this directory as workdir to gain access to script env + type_writer "๐Ÿ”จ---> Rebuilding with Docker Image Environment..." # Mount environment image on this directory as workdir to gain access to script env echo -e "${CYAN}======================================================${RESET}" echo "" @@ -350,14 +350,14 @@ if [[ -n ${UNTRUSTED_DOCKER_IMAGE:-} ]]; then echo -e "${CYAN}======================================================${RESET}" echo "" else - if [[ -n ${docker_image:-} ]]; then + if [[ -n ${env_image:-} ]]; then # Warning message for the users if their artifact was not built with a Docker Image, but a Docker Image was provided at command. - echo -e "${RED}[Warning] ${LIGHT_RED}Docker Image, $docker_image, provided, but artifact was not originally built on Docker Image${RESET}" + echo -e "${RED}[Warning] ${LIGHT_RED}Docker Image, $env_image, provided, but artifact was not originally built on Docker Image${RESET}" else echo "" # This is just for style. fi - # Run the build script locally without a docker image. + # Run the build script locally without an environment image. echo -e "${CYAN}======================================================${RESET}" type_writer "๐Ÿ’ป---> Rebuilding with local environment..." echo -e "${CYAN}======================================================${RESET}" From 773dd3730f32b4632183c6dee97fce457ffabfec Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 20:02:00 +0000 Subject: [PATCH 50/52] markdown lint Signed-off-by: Noah Elzner --- internal/builders/bazel/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/builders/bazel/README.md b/internal/builders/bazel/README.md index 346dad07ea..8bcb24a8d0 100644 --- a/internal/builders/bazel/README.md +++ b/internal/builders/bazel/README.md @@ -333,6 +333,7 @@ as follows on their [instructions](https://github.com/slsa-framework/slsa-verifi Verification can also be done through the passing the `--verify` flag to the rebuilder. ### Rebuilding + To rebuild your artifacts and check for reproducible builds use the Bazel Rebuilder, which takes in the following arguments on the command line. @@ -350,13 +351,14 @@ Arguments: | `--cleanup` | No | | Removes cloned repos (`source_uri` and `slsa-verifier`) as well as directory for rebuilt artifacts The rebuilder does the following: -~~~ + +``` 1. Verifies the provenance for the artifact to rebuild, 2. Parses out the attested build process from the provenance, 3. Clones the repo that produced it, 4. Rebuilds the inputted artifact with the attest build process, 5. Compares checksums for reproducibility -~~~ +``` An example usage of the rebuilder is the following command: `./rebuilder.sh --artifact_path== --prov_path= --source_uri= --builder_id= --env_image= --verify --verbose` @@ -366,4 +368,4 @@ must be built on an image from a registry originally during the Github Actions W If no image was provided or if the artifact was not originally built on an image, the rebuilding process will use the user's local environment, thus diminishing the chances of a reproducible rebuild due to non-determinism from the -different build environments. \ No newline at end of file +different build environments. From e47447c0988277b172b81d1b5647902fc03d06a6 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 20:05:13 +0000 Subject: [PATCH 51/52] lint Signed-off-by: Noah Elzner --- internal/builders/bazel/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/builders/bazel/README.md b/internal/builders/bazel/README.md index 8bcb24a8d0..7cf438fd77 100644 --- a/internal/builders/bazel/README.md +++ b/internal/builders/bazel/README.md @@ -352,13 +352,11 @@ Arguments: The rebuilder does the following: -``` 1. Verifies the provenance for the artifact to rebuild, 2. Parses out the attested build process from the provenance, 3. Clones the repo that produced it, 4. Rebuilds the inputted artifact with the attest build process, 5. Compares checksums for reproducibility -``` An example usage of the rebuilder is the following command: `./rebuilder.sh --artifact_path== --prov_path= --source_uri= --builder_id= --env_image= --verify --verbose` From c940cfd2db7579ce66eec45b39800febadc2de35 Mon Sep 17 00:00:00 2001 From: Noah Elzner Date: Fri, 11 Aug 2023 20:11:52 +0000 Subject: [PATCH 52/52] added output for actual artifacts dir name Signed-off-by: Noah Elzner --- .github/workflows/builder_bazel_slsa3.yml | 9 +++++++++ internal/builders/bazel/action.yml | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/builder_bazel_slsa3.yml b/.github/workflows/builder_bazel_slsa3.yml index 3b541ddb44..269a25a5c1 100644 --- a/.github/workflows/builder_bazel_slsa3.yml +++ b/.github/workflows/builder_bazel_slsa3.yml @@ -105,6 +105,15 @@ on: description: "SHA256 of the uploaded tarball of built artifacts." value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).artifacts-download-sha256 }} + artifacts-actual-name: + description: > + The name of the folder which contains the artifacts. + + After downloading artifacts-download-name and extracting + the folder.tgz from inside. A folder with artifacts with + this name will be extracted. + value: ${{ fromJSON(jobs.slsa-run.outputs.build-artifacts-outputs).artifacts-actual-name }} + jobs: slsa-setup: permissions: diff --git a/internal/builders/bazel/action.yml b/internal/builders/bazel/action.yml index 1f2fb828c1..647d28f631 100644 --- a/internal/builders/bazel/action.yml +++ b/internal/builders/bazel/action.yml @@ -49,6 +49,14 @@ outputs: artifacts-download-sha256: description: "SHA256 of the uploaded tarball of artifacts." value: ${{ steps.generate-artifacts.outputs.sha256 }} + artifacts-actual-name: + description: > + The name of the folder which contains the artifacts. + + After downloading artifacts-download-name and extracting + the folder.tgz from inside. A folder with artifacts with + this name will be extracted. + value: "bazel_builder_binaries_to_upload_to_gh_7bc972367cb286b7f36ab4457f06e369" runs: using: "composite"