Merge branch 'master' into feature/cat_recovery

Required-githooks: true

.github/workflows/ci2.yml

@@ -9,7 +9,7 @@ concurrency:
 
 jobs:
 
-  # Re-use the cache from the landing-builds workflow if available, if not then build the images
+  # reuse the cache from the landing-builds workflow if available, if not then build the images
   # from scratch, but do not save them.
   Build-and-test:
     name: Run DAOS/NLT tests

Jenkinsfile

@@ -317,7 +317,7 @@ pipeline {
                defaultValue: 'ci_vm1',
                description: 'Label to use for 1 VM node unit and RPM tests')
         string(name: 'CI_UNIT_VM1_NVME_LABEL',
-               defaultValue: 'bwx_vm1',
+               defaultValue: 'ci_ssd_vm1',
                description: 'Label to use for 1 VM node unit tests that need NVMe')
         string(name: 'FUNCTIONAL_VM_LABEL',
                defaultValue: 'ci_vm9',

SConstruct

@@ -317,8 +317,8 @@ def check_for_release_target():  # pylint: disable=too-many-locals
         try:
             remote.push(['refs/heads/{}'.format(branch)],
                         callbacks=MyCallbacks())
-        except pygit2.GitError as excpt:
-            print("Error pushing branch: {}".format(excpt))
+        except pygit2.GitError as err:
+            print("Error pushing branch: {}".format(err))
             Exit(1)
 
         print("Creating the PR...")

ci/functional/test_main.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+#
+#  Copyright 2020-2023 Intel Corporation.
+#
+#  SPDX-License-Identifier: BSD-2-Clause-Patent
 
 set -eux
 
@@ -12,14 +16,53 @@ test_tag="$TEST_TAG"
 tnodes=$(echo "$NODELIST" | cut -d ',' -f 1-"$NODE_COUNT")
 first_node=${NODELIST%%,*}
 
+cluster_reboot () {
+    # shellcheck disable=SC2029,SC2089
+    clush -B -S -o '-i ci_key' -l root -w "${tnodes}" reboot || true
+
+    # shellcheck disable=SC2029,SC2089
+    poll_cmd=( clush -B -S -o "-i ci_key" -l root -w "${tnodes}" )
+    poll_cmd+=( '"cat /etc/os-release"' )
+    reboot_timeout=900 # 15 minutes
+    retry_wait=10 # seconds
+    timeout=$((SECONDS + reboot_timeout))
+    while [ "$SECONDS" -lt "$timeout" ]; do
+      if "${poll_cmd[@]}"; then
+        return 0
+      fi
+      sleep ${retry_wait}
+    done
+    return 1
+}
+
+test_cluster() {
+    # Test that all nodes in the cluster are healthy
+    clush -B -S -o '-i ci_key' -l root -w "${tnodes}"   \
+        "OPERATIONS_EMAIL=${OPERATIONS_EMAIL}           \
+        FIRST_NODE=${first_node}                        \
+        TEST_RPMS=${TEST_RPMS}                          \
+        NODELIST=${tnodes}                              \
+        $(cat ci/functional/test_main_prep_node.sh)"
+}
+
 clush -B -S -o '-i ci_key' -l root -w "${first_node}" \
     "NODELIST=${NODELIST} $(cat ci/functional/setup_nfs.sh)"
 
-clush -B -S -o '-i ci_key' -l root -w "${tnodes}" \
-  "OPERATIONS_EMAIL=${OPERATIONS_EMAIL}                \
-   FIRST_NODE=${first_node}                            \
-   TEST_RPMS=${TEST_RPMS}                              \
-   $(cat ci/functional/test_main_prep_node.sh)"
+if ! test_cluster; then
+    # Sometimes a cluster reboot will fix the issue so try it once.
+    cluster_reboot
+    test_cluster
+fi
+
+# collect the _results.xml files from test_main_prep_nodes before they
+# may be deleted by pre-test cleanup.
+# Cannot use a wildcard for collection as that just ends up
+# with a wild card in the collected filename which just makes
+# things more confusing.
+rm -f ./hardware_prep_node_results.xml.* ./hardware_prep_nodde_*_results.xml
+clush -o '-i ci_key' -l root -w "$tnodes" \
+      --rcopy hardware_prep_node_results.xml
+# This results in file names with the node name as the suffix.
 
 # this is being mis-flagged as SC2026 where shellcheck.net is OK with it
 # shellcheck disable=SC2026
@@ -34,6 +77,7 @@ mkdir "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
 # set DAOS_TARGET_OVERSUBSCRIBE env here
 export DAOS_TARGET_OVERSUBSCRIBE=1
 rm -rf install/lib/daos/TESTING/ftest/avocado ./*_results.xml
+
 mkdir -p install/lib/daos/TESTING/ftest/avocado/job-results
 if $TEST_RPMS; then
     # shellcheck disable=SC2029
@@ -47,3 +91,16 @@ if $TEST_RPMS; then
 else
     ./ftest.sh "$test_tag" "$tnodes" "$FTEST_ARG"
 fi
+
+# Now rename the previously collected hardware test data for Jenkins
+# to use them for Junit processing.
+: "${STAGE_NAME:=}"
+mkdir -p "${STAGE_NAME}/hardware_prep/"
+for node in ${tnodes//,/ }; do
+    old_name="./hardware_prep_node_results.xml.$node"
+    new_name="${STAGE_NAME}/hardware_prep/${node}/results.xml"
+    if [ -e "$old_name" ]; then
+        mkdir -p "${STAGE_NAME}/hardware_prep/${node}"
+        mv "$old_name" "$new_name"
+    fi
+done