From 9dd778a4c2ec1e238950ab8609a83b6a19cf2b71 Mon Sep 17 00:00:00 2001 From: Abdullah Date: Mon, 2 Dec 2024 10:28:27 +0100 Subject: [PATCH 1/3] add DSS integration tests for CPU and NVIDIA GPU --- .../checkbox-dss-validation/bin/install-deps | 2 +- .../checkbox-provider-dss/bin/check_dss.sh | 6 ++ .../bin/check_dss_integration_tests.sh | 59 +++++++++++++++++++ .../checkbox-provider-dss/units/jobs.pxu | 47 +++++++++++++++ .../checkbox-provider-dss/units/test-plan.pxu | 4 ++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100755 contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh diff --git a/contrib/checkbox-dss-validation/bin/install-deps b/contrib/checkbox-dss-validation/bin/install-deps index 59f7263cdd..df39e80f00 100755 --- a/contrib/checkbox-dss-validation/bin/install-deps +++ b/contrib/checkbox-dss-validation/bin/install-deps @@ -80,7 +80,7 @@ main() { # intel_gpu_top command used for host-level GPU check # jq used for cases where jsonpath is insufficient for parsing json results echo -e "\nStep 3/4: Installing intel-gpu-tools" - DEBIAN_FRONTEND=noninteractive sudo apt install -y intel-gpu-tools jq + DEBIAN_FRONTEND=noninteractive sudo apt install -y python3-venv intel-gpu-tools jq echo -e "\nStep 4/4: Installing data-science-stack snap from channel $dss_snap_channel" sudo snap install data-science-stack --channel "$dss_snap_channel" diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh index 8a75be22a0..002821859b 100755 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh @@ -54,6 +54,11 @@ check_dss_can_remove_notebook() { fi } +check_dss_can_be_purged() { + dss purge + echo "Test success: dss purged." +} + help_function() { echo "This script is used for generic tests related to DSS" echo "Usage: check_dss.sh [args]..." @@ -78,6 +83,7 @@ main() { nvidia_gpu_acceleration_is_enabled) check_dss_status_contains "NVIDIA GPU acceleration: Enabled.*" ;; can_create_notebook) check_dss_can_create_notebook "${@:2}" ;; can_remove_notebook) check_dss_can_remove_notebook "${@:2}" ;; + can_be_purged) check_dss_can_be_purged ;; *) help_function ;; esac popd diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh new file mode 100755 index 0000000000..c9bc382132 --- /dev/null +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +DSS_REPO_CLONE_PATH="$HOME/data-science-stack" +DSS_REPO_REV="main" + +_ensure_microk8s_config_is_in_place() { + sudo microk8s.kubectl config view --raw | sudo tee "${SNAP_REAL_HOME}/.kube/config" >/dev/null +} + +_ensure_dss_repo_is_checked_out() { + if [ ! -d "$DSS_REPO_CLONE_PATH" ]; then + git clone https://github.com/canonical/data-science-stack.git "$DSS_REPO_CLONE_PATH" + fi + git -C "$DSS_REPO_CLONE_PATH" checkout "$DSS_REPO_REV" + echo "Current git branch for DSS repo: $(git branch --show-current)" + echo "Latest commit:" + git -C "$DSS_REPO_CLONE_PATH" log --name-status HEAD^..HEAD +} +_ensure_dss_python_env_is_setup() { + pushd "$DSS_REPO_CLONE_PATH" + if [ ! -d ".venv" ]; then + python3 -m venv .venv + fi + .venv/bin/pip install tox + popd +} + +check_dss_integration_tests_pass() { + pushd "$DSS_REPO_CLONE_PATH" + echo "starting DSS integration tests: $1" + .venv/bin/tox -e "$1" -- -vv -s + echo "Tests passed: DSS integration tests '$1'" + popd +} + +help_function() { + echo "This script is used for running integration tests from DSS" + echo "Usage: check_dss_integration_tests " + echo + echo "Test cases currently implemented:" + echo -e "\t" + echo -e "\t" +} + +main() { + _ensure_microk8s_config_is_in_place + _ensure_dss_repo_is_checked_out + _ensure_dss_python_env_is_setup + + case ${1} in + pass_on_cpu) check_dss_integration_tests_pass "integration" ;; + pass_on_nvidia_gpu) check_dss_integration_tests_pass "integration-gpu" ;; + *) help_function ;; + esac +} + +main "$@" diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu b/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu index 6f3a846865..a8fff3babe 100644 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu @@ -340,3 +340,50 @@ depends: dss/create_tensorflow_cuda_notebook _summary: Check that the Tensorflow CUDA notebook can be removed estimated_duration: 1m command: check_dss.sh can_remove_notebook tensorflow-cuda + +# DSS integration test jobs ################################################### + +id: dss/purge +category_id: dss-regress +flags: simple +imports: from com.canonical.certification import executable +requires: executable.name == 'dss' +depends: dss/initialize +_summary: Check that DSS can be purged +estimated_duration: 1m +command: check_dss.sh can_be_purged + +# NOTE: DSS Integration tests expect DSS to not be initialized +# hence they also depend on the dss/purge job + +id: dss_integration_tests/cpu +category_id: dss-regress +flags: simple +imports: + from com.canonical.certification import executable + from com.canonical.certification import package +requires: + executable.name == 'dss' + executable.name == 'python3' + package.name == 'python3-venv' +depends: dss/purge +_summary: Check that all DSS integration tests for CPU pass +estimated_duration: 1m +command: check_dss_integration_tests.sh can_be_purged pass_on_cpu + +id: dss_integration_tests/nvidia_gpu +category_id: dss-regress +flags: simple +imports: + from com.canonical.certification import executable + from com.canonical.certification import package +requires: + executable.name == 'dss' + executable.name == 'python3' + package.name == 'python3-venv' +depends: + dss/purge + nvidia_gpu_addon/validations_succeed +_summary: Check that all DSS integration tests for NVIDIA GPUs +estimated_duration: 1m +command: check_dss_integration_tests.sh can_be_purged pass_on_nvidia_gpu diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/units/test-plan.pxu b/contrib/checkbox-dss-validation/checkbox-provider-dss/units/test-plan.pxu index b1c0f99da4..1559a22ed4 100644 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/units/test-plan.pxu +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/units/test-plan.pxu @@ -35,7 +35,11 @@ include: dss/create_tensorflow_cuda_notebook cuda/tensorflow_can_use_cuda dss/remove_tensorflow_cuda_notebook + dss/purge + dss_integration_tests/cpu + dss_integration_tests/nvidia_gpu bootstrap_include: com.canonical.certification::executable com.canonical.certification::snap com.canonical.certification::graphics_card + com.canonical.certification::package From eeffa7b6fd4454aa1a54befaf13723dcc17455a9 Mon Sep 17 00:00:00 2001 From: Abdullah Date: Fri, 6 Dec 2024 16:23:16 +0100 Subject: [PATCH 2/3] fix calling integration scripts --- .../checkbox-provider-dss/units/jobs.pxu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu b/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu index a8fff3babe..9d1837505b 100644 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/units/jobs.pxu @@ -369,7 +369,7 @@ requires: depends: dss/purge _summary: Check that all DSS integration tests for CPU pass estimated_duration: 1m -command: check_dss_integration_tests.sh can_be_purged pass_on_cpu +command: check_dss_integration_tests.sh pass_on_cpu id: dss_integration_tests/nvidia_gpu category_id: dss-regress @@ -386,4 +386,4 @@ depends: nvidia_gpu_addon/validations_succeed _summary: Check that all DSS integration tests for NVIDIA GPUs estimated_duration: 1m -command: check_dss_integration_tests.sh can_be_purged pass_on_nvidia_gpu +command: check_dss_integration_tests.sh pass_on_nvidia_gpu From ba6233e155fa4c128a3c1afc2c88f2001b391250 Mon Sep 17 00:00:00 2001 From: Abdullah Date: Fri, 6 Dec 2024 16:26:22 +0100 Subject: [PATCH 3/3] exit with code 1 when forced to print help message this is to catch situations where the script is called with unsupported arguments. --- .../checkbox-provider-dss/bin/check_cuda.sh | 1 + .../checkbox-provider-dss/bin/check_dss.sh | 1 + .../checkbox-provider-dss/bin/check_dss_integration_tests.sh | 1 + .../checkbox-provider-dss/bin/check_intel.sh | 1 + .../checkbox-provider-dss/bin/check_notebook.sh | 1 + 5 files changed, 5 insertions(+) diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_cuda.sh b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_cuda.sh index 3bcfc8059d..4be57b0347 100755 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_cuda.sh +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_cuda.sh @@ -41,6 +41,7 @@ help_function() { echo "Test cases currently implemented:" echo -e "\t: check_nvidia_gpu_addon_can_be_enabled" echo -e "\t: check_nvidia_gpu_validations_succeed" + exit 1 } main() { diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh index 002821859b..9d321146e2 100755 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss.sh @@ -71,6 +71,7 @@ help_function() { echo -e "\t: check_dss_has_nvidia_gpu_acceleration_enabled" echo -e "\t: check_dss_can_create_notebook [args]" echo -e "\t: check_dss_can_remove_notebook " + exit 1 } main() { diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh index c9bc382132..c6596600d7 100755 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_dss_integration_tests.sh @@ -42,6 +42,7 @@ help_function() { echo "Test cases currently implemented:" echo -e "\t" echo -e "\t" + exit 1 } main() { diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_intel.sh b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_intel.sh index d0754acfea..82948ebe54 100755 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_intel.sh +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_intel.sh @@ -119,6 +119,7 @@ help_function() { echo -e "\t: check_at_least_one_intel_gpu_is_available" echo -e "\t: check_capacity_slots_for_intel_gpus_match" echo -e "\t: check_allocatable_slots_for_intel_gpus_match" + exit 1 } main() { diff --git a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_notebook.sh b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_notebook.sh index 2f49ebc7ee..b943a682ce 100755 --- a/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_notebook.sh +++ b/contrib/checkbox-dss-validation/checkbox-provider-dss/bin/check_notebook.sh @@ -33,6 +33,7 @@ help_function() { echo -e "\t\t\t- verifying_tensorflow_can_use_xpu" echo -e "\t\t\t- verifying_pytorch_can_use_cuda" echo -e "\t\t\t- verifying_tensorflow_can_use_cuda" + exit 1 } main() {