Resolved comment

Author: vipnydav

perf-benchmarking-for-releases/README.md

@@ -49,6 +49,8 @@ Before running the benchmarks, ensure you have:
 The main script to run the benchmarks is `run-benchmarks.sh`.  
 It should be executed from the `perf-benchmarking-for-releases` directory.
 
+**Note:** This framework currently only supports benchmarking against regional GCS buckets with a flat object namespace (i.e., non-hierarchical).
+
 ### Syntax
 
 ```bash
@@ -62,13 +64,13 @@ bash run-benchmarks.sh <GCSFUSE_VERSION> <PROJECT_ID> <REGION> <MACHINE_TYPE> <I
 - `<REGION>`: The GCP region where the VM and GCS buckets will be created (e.g., `us-south1`).
 - `<MACHINE_TYPE>`: The GCE machine type for the benchmark VM (e.g., `n2-standard-96`). This script supports attaching 16 local NVMe SSDs (375GB each) for LSSD-supported machine types.
    - **Note:** If your machine type supports LSSD but is not included in the `LSSD_SUPPORTED_MACHINES` array within `run-benchmarks.sh` script, you may need to manually add it to ensure LSSDs are attached.
-- `<IMAGE_FAMILY>`: The image family for the VM (e.g., `ubuntu-2204-lts`).
+- `<IMAGE_FAMILY>`: The image family for the VM (e.g., `ubuntu-2504-amd64`).
 - `<IMAGE_PROJECT>`: The image project for the VM (e.g., `ubuntu-os-cloud`).
 
 ### Example:
 
 ```bash
-bash run-benchmarks.sh v2.12.0 gcs-fuse-test us-south1 n2-standard-96 ubuntu-2204-lts ubuntu-os-cloud
+bash run-benchmarks.sh master gcs-fuse-test us-south1 n2-standard-96 ubuntu-2504-amd64 ubuntu-os-cloud
 ```
 
 ---

perf-benchmarking-for-releases/run-benchmarks.sh

@@ -22,7 +22,7 @@ if [ "$#" -ne 6 ]; then
     echo "This script should be run from the 'perf-benchmarking-for-releases' directory."
     echo ""
     echo "Example:"
-    echo "  bash run-benchmarks.sh v2.12.0 gcs-fuse-test us-south1 n2-standard-96 ubuntu-2204-lts ubuntu-os-cloud"
+    echo "  bash run-benchmarks.sh master gcs-fuse-test us-south1 n2-standard-96 ubuntu-2504-amd64 ubuntu-os-cloud"
     exit 1
 fi
 
@@ -50,6 +50,7 @@ UNIQUE_ID="${TIMESTAMP}-${RAND_SUFFIX}"
 VM_NAME="gcsfuse-perf-benchmark-${UNIQUE_ID}"
 GCS_BUCKET_WITH_FIO_TEST_DATA="gcsfuse-release-benchmark-data-${UNIQUE_ID}"
 RESULTS_BUCKET_NAME="gcsfuse-release-benchmarks-results"
+BQ_TABLE="gcs-fuse-test-ml.gke_test_tool_outputs.fio_outputs"
 RESULT_PATH="gs://${RESULTS_BUCKET_NAME}/${GCSFUSE_VERSION}-${UNIQUE_ID}"
 
 
@@ -108,8 +109,8 @@ echo "Creating GCS test data bucket: gs://${GCS_BUCKET_WITH_FIO_TEST_DATA} in re
 gcloud storage buckets create "gs://${GCS_BUCKET_WITH_FIO_TEST_DATA}" --project="${PROJECT_ID}" --location="${REGION}"
 
 # Upload FIO job files to the results bucket for the VM to download
-echo "Uploading all .fio job files from local 'fio-job-files/' directory to ${RESULT_PATH}/fio_job_files/..."
-gcloud storage cp fio_job_files/*.fio "${RESULT_PATH}/fio-job-files/"
+echo "Uploading all .fio job files from local 'fio-job-files/' directory to ${RESULT_PATH}/fio-job-files/..."
+gcloud storage cp fio-job-files/*.fio "${RESULT_PATH}/fio-job-files/"
 echo "FIO job files uploaded."
 
 # Get the project number
@@ -119,11 +120,14 @@ PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectN
 STS_ACCOUNT="project-${PROJECT_NUMBER}@storage-transfer-service.iam.gserviceaccount.com"
 
 # Grant the service account 'roles/storage.admin' permissions on the newly created bucket
+# This allows the service account to manage the bucket and perform transfers
 gcloud storage buckets add-iam-policy-binding "gs://${GCS_BUCKET_WITH_FIO_TEST_DATA}" \
   --member="serviceAccount:${STS_ACCOUNT}" \
   --role="roles/storage.admin"
 
-# Use storage transfer job to copy test data from a fixed GCS bucket.
+# Since file generation with fio is painfully slow, we will use storage transfer
+# job to transfer test data from a fixed GCS bucket to the newly created bucket.
+# Note : We need to copy only read data.
 echo "Creating storage transfer job to copy read data to gs://${GCS_BUCKET_WITH_FIO_TEST_DATA}..."
 gcloud transfer jobs create \
   gs://gcsfuse-release-benchmark-fio-data \
@@ -158,16 +162,18 @@ SUCCESS_FILE_PATH="${RESULT_PATH}/success.txt"
 LOG_FILE_PATH="${RESULT_PATH}/benchmark_run.log"
 SLEEP_TIME=300  # 5 minutes
 sleep "$SLEEP_TIME"
+#max 18 retries amounting to ~1hr30mins time
 MAX_RETRIES=18
 
 for ((i=1; i<=MAX_RETRIES; i++)); do
     if gcloud storage objects describe "${SUCCESS_FILE_PATH}" &> /dev/null; then
         echo "Benchmarks completed. success.txt found."
-        echo "Results are available in BigQuery: gcs-fuse-test-ml.gke_test_tool_outputs.fio_outputs"
+        echo "Results are available in BigQuery: ${BQ_TABLE}"
         echo "Benchmark log file: $LOG_FILE_PATH"
         exit 0
     fi
 
+    # Check for early failure indicators
     if gcloud storage objects describe "${RESULT_PATH}/details.txt" &> /dev/null || \
        gcloud storage objects describe "$LOG_FILE_PATH" &> /dev/null; then
         echo "Benchmark log or details.txt found, but success.txt is missing. Possible error in benchmark execution."
@@ -179,7 +185,9 @@ for ((i=1; i<=MAX_RETRIES; i++)); do
     sleep "$SLEEP_TIME"
 done
 
-
+# Failure case: success.txt was not found after retries
 echo "Timed out waiting for success.txt after $((MAX_RETRIES * SLEEP_TIME / 60)) minutes. Perhaps there is some error."
 echo "Benchmark log file (for troubleshooting): $LOG_FILE_PATH"
 exit 1
+
+# The trap command will handle the cleanup on script exit.

perf-benchmarking-for-releases/starter-script.sh

@@ -33,9 +33,10 @@ else
     exit 1
 fi
 
+# Install common dependencies before adding starterscriptuser
 if [[ "$OS_FAMILY" == "debian_ubuntu" ]]; then
     sudo apt-get update
-    sudo apt-get install -y wget git fio libaio-dev gcc make mdadm build-essential python3-setuptools python3-crcmod python3-pip fuse jq bc procps gawk
+    sudo apt-get install -y wget git fio libaio-dev gcc make mdadm build-essential python3-setuptools python3-crcmod python3-pip python3-venv fuse jq bc procps gawk
 elif [[ "$OS_FAMILY" == "rhel_centos" ]]; then
     sudo yum makecache
     sudo yum -y install git fio fuse libaio libaio-devel gcc make mdadm redhat-rpm-config python3-devel python3-setuptools python3-pip jq bc procps-ng wget gawk
@@ -60,6 +61,8 @@ sudo -u starterscriptuser OS_FAMILY="$OS_FAMILY" bash <<'EOF'
 set -x
 set -e
 
+UPLOAD_FAILED=false
+
 # Function to monitor GCSFuse CPU and memory usage
 monitor_gcsfuse_usage() {
   local log_file="$1"
@@ -122,6 +125,7 @@ monitor_gcsfuse_usage() {
 }
 
 BENCHMARK_LOG_FILE="/tmp/benchmark_run.log"
+# Redirect stdout and stderr to BENCHMARK_LOG_FILE and also to original stdout/stderr
 exec > >(tee -a "$BENCHMARK_LOG_FILE") 2>&1
 
 cleanup() {
@@ -144,6 +148,7 @@ cd ~/
 echo "Current directory: $(pwd)"
 echo "User: $(whoami)"
 
+# Fetch metadata parameters
 GCSFUSE_VERSION=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/GCSFUSE_VERSION" -H "Metadata-Flavor: Google")
 GCS_BUCKET_WITH_FIO_TEST_DATA=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/GCS_BUCKET_WITH_FIO_TEST_DATA" -H "Metadata-Flavor: Google")
 RESULT_PATH=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/RESULT_PATH" -H "Metadata-Flavor: Google")
@@ -153,6 +158,7 @@ VM_NAME=$(hostname)
 UNIQUE_ID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/UNIQUE_ID" -H "Metadata-Flavor: Google")
 GCSFUSE_MOUNT_OPTIONS_STR="implicit-dirs"
 
+# Determine system architecture
 ARCHITECTURE=""
 if [[ "$OS_FAMILY" == "debian_ubuntu" ]]; then
     ARCHITECTURE=$(dpkg --print-architecture)
@@ -167,9 +173,13 @@ elif [[ "$OS_FAMILY" == "rhel_centos" ]]; then
         exit 1
     fi
 fi
+
+# Install Go
 wget -nv --tries=3 --waitretry=5 -O go_tar.tar.gz "https://go.dev/dl/go1.24.0.linux-${ARCHITECTURE}.tar.gz"
 sudo tar -C /usr/local -xzf go_tar.tar.gz
 export PATH=$PATH:/usr/local/go/bin
+
+# Clone and build gcsfuse
 git clone https://github.com/GoogleCloudPlatform/gcsfuse.git
 cd gcsfuse
 git checkout "$GCSFUSE_VERSION"
@@ -182,15 +192,19 @@ GCSFUSE_BIN="$CURR_DIR/gcsfuse/gcsfuse"
 MNT="$CURR_DIR/$MOUNT_POINT"
 SSD_MOUNT_DIR="/mnt/disks/local_ssd"
 FIO_JOB_DIR="/tmp/fio_jobs"
+
+# Download all FIO job spec files
 mkdir -p "$FIO_JOB_DIR"
-gcloud storage cp "${RESULT_PATH}/fio_job_files/*.fio" "$FIO_JOB_DIR/"
+gcloud storage cp "${RESULT_PATH}/fio-job-files/*.fio" "$FIO_JOB_DIR/"
 
+# Capture versions
 {
     echo "GCSFuse version: $GCSFUSE_VERSION"
     echo "Go version     : $(go version)"
     echo "FIO version    : $(fio --version)"
 } >> details.txt
 
+# Create LSSD if enabled
 if [[ "$LSSD_ENABLED" == "true" ]]; then
     LSSD_DEVICES=()
     for i in {0..15}; do
@@ -213,7 +227,12 @@ if [[ "$LSSD_ENABLED" == "true" ]]; then
 fi
 
 git clone --single-branch --branch fio-to-bigquery https://github.com/GoogleCloudPlatform/gcsfuse-tools.git
-python3 -m pip install --user -r gcsfuse-tools/perf_benchmarking_for_releases/requirements.txt
+cd gcsfuse-tools
+
+python3 -m venv py_venv
+source py_venv/bin/activate
+python3 -m pip install -r perf-benchmarking-for-releases/requirements.txt
+
 IFS=',' read -r -a GCSFUSE_FLAGS_ARRAY <<< "$GCSFUSE_MOUNT_OPTIONS_STR"
 GCSFUSE_FLAGS=()
 for flag in "${GCSFUSE_FLAGS_ARRAY[@]}"; do
@@ -255,6 +274,7 @@ for master_fio_file in "$FIO_JOB_DIR"/*.fio; do
         [[ "$LSSD_ENABLED" == "true" ]] && reformat_and_remount_lssd
         sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
 
+        # Mount GCS bucket using gcsfuse
         mkdir -p "$MNT"
         "$GCSFUSE_BIN" "${GCSFUSE_FLAGS[@]}" "$GCS_BUCKET_WITH_FIO_TEST_DATA" "$MNT"
         
@@ -281,26 +301,40 @@ for master_fio_file in "$FIO_JOB_DIR"/*.fio; do
         read -r LOWEST_CPU HIGHEST_CPU <<< $(gawk 'BEGIN{min="inf";max="-inf"} {if($2<min)min=$2; if($2>max)max=$2} END{if(min=="inf")print "0.0 0.0"; else print min, max}' "$monitor_log")
         read -r LOWEST_MEM HIGHEST_MEM <<< $(gawk 'BEGIN{min="inf";max="-inf"} {if($3<min)min=$3; if($3>max)max=$3} END{if(min=="inf")print "0 0"; else print min, max}' "$monitor_log")
 
-        python3 gcsfuse-tools/perf_benchmarking_for_releases/upload_fio_output_to_bigquery.py \
+        if python3 perf-benchmarking-for-releases/upload_fio_output_to_bigquery.py \
           --result-file "$RESULT_FILE" \
           --fio-job-file "$single_fio_file" \
           --master-fio-file "$master_fio_file" \
           --lowest-cpu "$LOWEST_CPU" \
           --highest-cpu "$HIGHEST_CPU" \
           --lowest-mem "$LOWEST_MEM" \
           --highest-mem "$HIGHEST_MEM" \
-          --gcsfuse-mount-options "$GCSFUSE_MOUNT_OPTIONS_STR"
-
-        rm -f "$RESULT_FILE" "$monitor_log"
+          --gcsfuse-mount-options "$GCSFUSE_MOUNT_OPTIONS_STR"; then
+            echo "Successfully uploaded results to BigQuery for job: $job_file_basename"
+            rm -f "$RESULT_FILE" "$monitor_log"
+        else
+            echo "Warning: Failed to upload results to BigQuery for job: $job_file_basename. Uploading monitor log to GCS for debugging."
+            gcloud storage cp "$monitor_log" "${RESULTS_SUBDIR_PATH}/" || echo "Warning: Failed to upload monitor log for ${job_file_basename}"
+            UPLOAD_FAILED=true
+        fi
     done
 
     rm -rf "$SPLIT_DIR"
 done
 
-touch success.txt
-gcloud storage cp success.txt "$RESULT_PATH"
-rm success.txt
+cd ..
+
+if [[ "$UPLOAD_FAILED" == "false" ]]; then
+    # All tests ran successfully; create a success.txt file in GCS
+    touch success.txt
+    gcloud storage cp success.txt "$RESULT_PATH"
+    rm success.txt
+else
+    echo "One or more BigQuery uploads failed. Not creating success.txt to indicate benchmark failure."
+fi
 
 EOF
 
 echo "Starter script finished execution on VM."
+
+# The trap command will handle the cleanup on script exit.

perf-benchmarking-for-releases/upload_fio_output_to_bigquery.py

@@ -1,6 +1,22 @@
+#!/bin/bash
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 import datetime
 import argparse
+import sys
 from google.cloud import bigquery
 import requests
 
@@ -214,5 +230,6 @@ def fetch_metadata(attribute):
     errors = client.insert_rows_json(full_table_id, [row_to_insert])
     if errors:
         print("Errors inserting rows:", errors)
+        sys.exit(1)
     else:
         print(f"Inserted 1 row for job '{jobname}' into {full_table_id}")