CI: improve test_jenkins.sh error reporting

openucx · Sep 11, 2024 · 86215e9 · 86215e9
1 parent a6d151b
commit 86215e9
Showing 1 changed file with 35 additions and 19 deletions.
diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh
@@ -599,7 +599,7 @@ run_mpi_tests() {
 			save_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
 			export LD_LIBRARY_PATH=${ucx_inst}/lib:${MPI_HOME}/lib:${prev_LD_LIBRARY_PATH}
 
-			build release --disable-gtest --with-mpi
+			build release --disable-gtest --with-mpi || return 1
 
 			# check whether installation is valid (it compiles examples at least)
 			$MAKEP installcheck
@@ -983,7 +983,8 @@ run_gtest_armclang() {
 				CC=armclang \
 				CXX=armclang++ \
 				CFLAGS="${ARMCLANG_CFLAGS}" \
-				--without-go
+				--without-go \
+                || return 1
 
 			run_gtest "armclang"
 		else
@@ -1043,7 +1044,7 @@ run_ucx_tl_check() {
 # Run release mode tests
 #
 run_release_mode_tests() {
-	build release --enable-gtest
+	build release --enable-gtest || return 1
 	test_profiling
 	test_ucs_load
 	test_ucp_dlopen
@@ -1057,7 +1058,7 @@ run_release_mode_tests() {
 run_nt_buffer_transfer_tests() {
     if lscpu | grep -q 'AuthenticAMD'
     then
-	    build release --enable-gtest --enable-optimizations
+	    build release --enable-gtest --enable-optimizations || return 1
 	    echo "==== Running nt_buffer_transfer tests ===="
 	    ./test/gtest/gtest --gtest_filter="test_arch.nt_buffer_transfer_*"
     fi
@@ -1087,7 +1088,7 @@ run_tests() {
 	run_mpi_tests
 
 	# build for devel tests and gtest
-	build devel --enable-gtest
+	build devel --enable-gtest || return 1
 
 	# devel mode tests
 	do_distributed_task 0 4 test_unused_env_var
@@ -1121,7 +1122,7 @@ run_tests() {
 
 run_test_proto_disable() {
 	# build for devel tests and gtest
-	build devel --enable-gtest
+	build devel --enable-gtest || return 1
 
 	export UCX_PROTO_ENABLE=n
 
@@ -1130,7 +1131,7 @@ run_test_proto_disable() {
 }
 
 run_asan_check() {
-	build devel --enable-gtest --enable-asan --without-valgrind
+	build devel --enable-gtest --enable-asan --without-valgrind || return 1
 	run_gtest "default"
 }
 
@@ -1147,7 +1148,7 @@ run_valgrind_check() {
 	fi
 
 	echo "==== Run valgrind tests ===="
-	build devel --enable-gtest
+	build devel --enable-gtest || return 1
 	run_gtest_make "default" test_valgrind
 	module unload tools/valgrind-3.12.0
 }
@@ -1158,15 +1159,30 @@ try_load_cuda_env
 if [ "$RUN_TESTS" == "yes" ]
 then
     check_machine
-    set_ucx_common_test_env
-
-    if [[ "$PROTO_ENABLE" == "no" ]]; then
-        run_test_proto_disable
-    elif [[ "$ASAN_CHECK" == "yes" ]]; then
-        run_asan_check
-    elif [[ "$VALGRIND_CHECK" == "yes" ]]; then
-        run_valgrind_check
-    else
-        run_tests
-    fi
+     set_ucx_common_test_env
+
+     if [[ "$PROTO_ENABLE" == "no" ]]; then
+         run_test_proto_disable || {
+             set +x
+             azure_log_error "Proto V1 test failed"
+             azure_log_error "To debug, rerun with: PROTO_ENABLE=no $0"
+             exit 1
+         }
+     elif [[ "$ASAN_CHECK" == "yes" ]]; then
+         run_asan_check || {
+             set +x
+             azure_log_error "AddressSanitaizer check failed"
+             azure_log_error	"To debug, rerun with: ASAN_CHECK=yes $0"
+             exit 1
+         }
+     elif [[ "$VALGRIND_CHECK" == "yes" ]]; then
+         run_valgrind_check || {
+             set +x
+             azure_log_error "Valgrind test failed"
+             azure_log_error "To debug, rerun with: VALGRIND_CHECK=yes $0"
+             exit 1
+         }
+     else
+         run_tests
+     fi
 fi