[BES notebooks] - Fix the issue with cli-batch-score (#3728)

Author: trimicrosoft

.github/workflows/cli-scripts-batch-score.yml

@@ -27,6 +27,8 @@ jobs:
     steps:
     - name: check out repo
       uses: actions/checkout@v2
+      with:
+        lfs: true
     - name: azure login
       uses: azure/login@v1
       with:

cli/batch-score.sh

@@ -37,9 +37,18 @@ az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NA
 
 sleep 60
 
-echo "Invoking batch endpoint with public URI (MNIST)"
+echo "Extracting MNIST data from zip file"
+# <extract_data>
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+REPO_ROOT="$(dirname "$SCRIPT_DIR")"
+TEMP_DATA_PATH="$REPO_ROOT/temp_mnist_data"
+mkdir -p $TEMP_DATA_PATH
+unzip -q "$REPO_ROOT/assets/batch-endpoints-data/mnist-batch/data.zip" -d $TEMP_DATA_PATH
+# </extract_data>
+
+echo "Invoking batch endpoint with local data (MNIST)"
 # <start_batch_scoring_job>
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/mnist/sample --input-type uri_folder --query name -o tsv)
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input $TEMP_DATA_PATH/data --input-type uri_folder --query name -o tsv)
 # </start_batch_scoring_job>
 
 echo "Showing job detail"
@@ -71,13 +80,13 @@ fi
 echo "Invoke batch endpoint with specific output file name"
 # <start_batch_scoring_job_configure_output_settings>
 export OUTPUT_FILE_NAME=predictions_`echo $RANDOM`.csv
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/mnist/sample --input-type uri_folder --output-path azureml://datastores/workspaceblobstore/paths/$ENDPOINT_NAME --set output_file_name=$OUTPUT_FILE_NAME --query name -o tsv)
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input $TEMP_DATA_PATH/data --input-type uri_folder --output-path azureml://datastores/workspaceblobstore/paths/$ENDPOINT_NAME --set output_file_name=$OUTPUT_FILE_NAME --query name -o tsv)
 # </start_batch_scoring_job_configure_output_settings>
 
 echo "Invoke batch endpoint with specific overwrites"
 # <start_batch_scoring_job_overwrite>
 export OUTPUT_FILE_NAME=predictions_`echo $RANDOM`.csv
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/mnist/sample --input-type uri_folder --mini-batch-size 20 --instance-count 5 --query name -o tsv)
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input $TEMP_DATA_PATH/data --input-type uri_folder --mini-batch-size 20 --instance-count 5 --query name -o tsv)
 # </start_batch_scoring_job_overwrite>
 
 echo "Stream job detail"
@@ -114,7 +123,7 @@ az ml batch-deployment create --file endpoints/batch/deploy-models/mnist-classif
 echo "Invoke batch endpoint with public data"
 # <test_new_deployment>
 DEPLOYMENT_NAME="mnist-keras-dpl"
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --deployment-name $DEPLOYMENT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/mnist/sample --input-type uri_folder --query name -o tsv)
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --deployment-name $DEPLOYMENT_NAME --input $TEMP_DATA_PATH/data --input-type uri_folder --query name -o tsv)
 # </test_new_deployment>
 
 echo "Show job detail"
@@ -155,7 +164,7 @@ az ml batch-endpoint show --name $ENDPOINT_NAME --query "{Name:name, Defaults:de
 
 echo "Invoke batch endpoint with the new default deployment with public URI"
 # <test_new_default_deployment>
-JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/mnist/sample --input-type uri_folder --query name -o tsv)
+JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input $TEMP_DATA_PATH/data --input-type uri_folder --query name -o tsv)
 # </test_new_default_deployment>
 
 echo "Stream job logs to console"
@@ -184,21 +193,32 @@ echo "Get Scoring URI"
 SCORING_URI=$(az ml batch-endpoint show --name $ENDPOINT_NAME --query scoring_uri -o tsv)
 # </get_scoring_uri>
 
+echo "Upload data to datastore for REST API call"
+# <upload_data_to_datastore>
+DATA_ASSET_NAME="mnist-batch-data-$(date +%s)"
+az ml data create --name $DATA_ASSET_NAME --version 1 --type uri_folder --path $TEMP_DATA_PATH/data
+# </upload_data_to_datastore>
+
 echo "Get Token"
 # <get_token>
 AUTH_TOKEN=$(az account get-access-token --resource https://ml.azure.com --query accessToken -o tsv)
 # </get_token>
 
 echo "Invoke batch endpoint with REST API call"
 # <start_batch_scoring_job_rest>
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+RESOURCE_GROUP=$(az configure -l --query "[?name=='group'].value" -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
+DATA_URI="azureml://subscriptions/$SUBSCRIPTION_ID/resourcegroups/$RESOURCE_GROUP/workspaces/$WORKSPACE/data/$DATA_ASSET_NAME/versions/1"
+
 RESPONSE=$(curl --location --request POST "$SCORING_URI" \
 --header "Authorization: Bearer $AUTH_TOKEN" \
 --header "Content-Type: application/json" \
 --data-raw "{
   \"properties\": {
     \"dataset\": {
       \"dataInputType\": \"DataUrl\",
-      \"Path\": \"https://azuremlexampledata.blob.core.windows.net/data/mnist/sample\"
+      \"Path\": \"$DATA_URI\"
     }
   }
 }")
@@ -253,3 +273,8 @@ az ml batch-deployment delete --name nonmlflowdp --endpoint-name $ENDPOINT_NAME
 # <delete_endpoint>
 az ml batch-endpoint delete --name $ENDPOINT_NAME --yes
 # </delete_endpoint>
+
+echo "Cleaning up temporary data folder"
+# <cleanup_temp_data>
+rm -rf $TEMP_DATA_PATH
+# </cleanup_temp_data>

cli/endpoints/batch/deploy-models/mnist-classifier/deployment-keras/deployment.yml

@@ -11,7 +11,7 @@ code_configuration:
   scoring_script: batch_driver.py
 environment:
   name: batch-tensorflow-py38
-  image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest
+  image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest
   conda_file: environment/conda.yaml
 compute: azureml:batch-cluster
 resources:

cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml

@@ -11,7 +11,7 @@ code_configuration:
   scoring_script: batch_driver.py
 environment:
   name: batch-torch-py38
-  image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest
+  image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu22.04:latest
   conda_file: environment/conda.yaml
 compute: azureml:batch-cluster
 resources: