SRE-3584 Build: Improve Hardware test reporting

Jenkinsfile

@@ -1,9 +1,10 @@
 #!/usr/bin/groovy
 /* groovylint-disable-next-line LineLength */
 /* groovylint-disable DuplicateMapLiteral, DuplicateNumberLiteral */
-/* groovylint-disable DuplicateStringLiteral, NestedBlockDepth, VariableName */
+/* groovylint-disable DuplicateStringLiteral, NestedBlockDepth */
+/* groovylint-disable ParameterName, VariableName */
 /* Copyright 2019-2024 Intel Corporation
- * Copyright 2025 Hewlett Packard Enterprise Development LP
+ * Copyright 2025-2026 Hewlett Packard Enterprise Development LP
  * All rights reserved.
  *
  * This file is part of the DAOS Project. It is subject to the license terms
@@ -133,12 +134,16 @@ String vm9_label(String distro) {
                                                           def_val: params.FUNCTIONAL_VM_LABEL))
 }
 
-void rpm_test_post(String stage_name, String node) {
+void rpm_test_post(String stageName, String node) {
+    // Extract first node from coma-delimited list
+    String firstNode = node.split(',')[0].trim()
     sh label: 'Fetch and stage artifacts',
-       script: 'hostname; ssh -i ci_key jenkins@' + node + ' ls -ltar /tmp; mkdir -p "' +  env.STAGE_NAME + '/" && ' +
-               'scp -i ci_key jenkins@' + node + ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*} "' +
-               env.STAGE_NAME + '/"'
-    archiveArtifacts artifacts: env.STAGE_NAME + '/**'
+       script: 'hostname; ssh -i ci_key jenkins@' + firstNode +
+               ' ls -ltar /tmp; mkdir -p "' +  stageName + '/" && ' +
+               'scp -i ci_key jenkins@' + firstNode +
+               ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*} "' +
+               stageName + '/"'
+    archiveArtifacts artifacts: stageName + '/**'
     job_status_update()
 }
 
@@ -148,7 +153,7 @@ void rpm_test_post(String stage_name, String node) {
 Map update_default_commit_pragmas() {
     String default_pragmas_str = sh(script: 'ci/gen_commit_pragmas.py --target origin/' + target_branch,
                                     returnStdout: true).trim()
-    println("pragmas from gen_commit_pragmas.py:")
+    println('pragmas from gen_commit_pragmas.py:')
     println(default_pragmas_str)
     if (default_pragmas_str) {
         updatePragmas(default_pragmas_str, false)
@@ -851,7 +856,7 @@ pipeline {
                             unitTestPost artifacts: ['nlt_logs/'],
                                          testResults: 'nlt-junit.xml',
                                          always_script: 'ci/unit/test_nlt_post.sh',
-					 referenceJobName: 'daos-stack/daos/release%252F2.6',
+                                         referenceJobName: 'daos-stack/daos/release%252F2.6',
                                          valgrind_stash: 'el8-gcc-nlt-memcheck'
                             recordIssues enabledForFailure: true,
                                          failOnError: false,

ci/functional/test_main.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 #  Copyright 2020-2024 Intel Corporation.
-#  Copyright 2025 Hewlett Packard Enterprise Development LP
+#  Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 #
 #  SPDX-License-Identifier: BSD-2-Clause-Patent
 #
@@ -14,6 +14,13 @@ fi
 
 test_tag="$TEST_TAG"
 
+: "${NODELIST:=localhost}"
+: "${TEST_RPMS:=false}"
+: "${STAGE_NAME:=unknown}"
+
+def_node_count="$(nodeset -c "$NODELIST")"
+: "${NODE_COUNT:=$def_node_count}"
+
 tnodes=$(echo "$NODELIST" | cut -d ',' -f 1-"$NODE_COUNT")
 first_node=${NODELIST%%,*}
 
@@ -42,14 +49,17 @@ cluster_reboot () {
 test_cluster() {
     # Test that all nodes in the cluster are healthy
     clush -B -S -o '-i ci_key' -l root -w "${tnodes}"   \
-        "OPERATIONS_EMAIL=${OPERATIONS_EMAIL}           \
+        "OPERATIONS_EMAIL=${OPERATIONS_EMAIL:-}         \
         FIRST_NODE=${first_node}                        \
         TEST_RPMS=${TEST_RPMS}                          \
         NODELIST=${tnodes}                              \
         BUILD_URL=\"${BUILD_URL:-Unknown in GHA}\"      \
-        STAGE_NAME=\"$STAGE_NAME\"                      \
+        STAGE_NAME=\"${STAGE_NAME}\"                    \
         JENKINS_URL=\"${JENKINS_URL:-}\"                \
         DAOS_DEVOPS_EMAIL=\"${DAOS_DEVOPS_EMAIL:-}\"    \
+        DAOS_INFINIBAND=${DAOS_INFINIBAND:-}            \
+        DAOS_NVME=${DAOS_NVME:-}                        \
+        DAOS_PMEM=${DAOS_PMEM:-}                        \
         $(cat ci/functional/test_main_prep_node.sh)"
 }
 
@@ -65,7 +75,7 @@ if ! test_cluster; then
             echo "Hardware test failed again after reboot"
         fi
     else
-        echo "Cluster reboot failed"        
+        echo "Cluster reboot failed"
     fi
 else
     hardware_ok=true
@@ -88,6 +98,7 @@ trap 'clush -B -S -o "-i ci_key" -l root -w "${tnodes}" '\
 
 # Setup the Jenkins build artifacts directory before running the tests to ensure
 # there is enough disk space to report the results.
+# Even though STAGE_NAME forced to be set, shellcheck wants this syntax.
 rm -rf "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
 mkdir "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
 
@@ -98,24 +109,23 @@ rm -rf install/lib/daos/TESTING/ftest/avocado ./*_results.xml
 mkdir -p install/lib/daos/TESTING/ftest/avocado/job-results
 
 if "$hardware_ok"; then
-    if $TEST_RPMS; then
+    if "$TEST_RPMS"; then
         # shellcheck disable=SC2029
-        ssh -i ci_key -l jenkins "${first_node}"   \
-          "TEST_TAG=\"$test_tag\"                  \
-           TNODES=\"$tnodes\"                      \
-           FTEST_ARG=\"${FTEST_ARG:-}\"            \
-           WITH_VALGRIND=\"${WITH_VALGRIND:-}\"    \
-           STAGE_NAME=\"$STAGE_NAME\"              \
-           DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\"        \
+        ssh -i ci_key -l jenkins "${first_node}"      \
+          "TEST_TAG=\"$test_tag\"                     \
+           TNODES=\"$tnodes\"                         \
+           FTEST_ARG=\"${FTEST_ARG:-}\"               \
+           WITH_VALGRIND=\"${WITH_VALGRIND:-}\"       \
+           STAGE_NAME=\"${STAGE_NAME}\"               \
+           DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \
            $(cat ci/functional/test_main_node.sh)"
     else
-        ./ftest.sh "$test_tag" "$tnodes" "$FTEST_ARG"
+        ./ftest.sh "$test_tag" "$tnodes" "${FTEST_ARG:-}"
     fi
 fi
 
 # Now rename the previously collected hardware test data for Jenkins
 # to use them for Junit processing.
-: "${STAGE_NAME:=}"
 mkdir -p "${STAGE_NAME}/hardware_prep/"
 for node in ${tnodes//,/ }; do
     old_name="./hardware_prep_node_results.xml.$node"