DAOS-14369 test: Rework pydaos and avocado install/setup process.

.dockerignore

@@ -24,7 +24,9 @@
 !ftest.sh
 !site_scons
 !.clang-format
-!requirements.txt
+!requirements-build.txt
+!requirements-utest.txt
+!requirements-ftest.txt
 
 # Now disallow extra files which may be present but are not required.
 # Include the rdb repo as well.

.github/workflows/landing-builds.yml

@@ -14,7 +14,8 @@ on:
       - 'utils/scripts/helpers/*'
       - utils/ci
       - ci
-      - requirements.txt
+      - requirements-build.txt
+      - requirements-utest.txt
 
 jobs:
 

.github/workflows/linting.yml

@@ -26,7 +26,7 @@ jobs:
           python-version: '3'
       - uses: isort/isort-action@master
         with:
-          requirementsFiles: "requirements.txt utils/cq/requirements.txt"
+          requirementsFiles: "requirements.txt"
       - name: Run on SConstruct file.
         run: isort --check-only SConstruct
       - name: Run on build files.

.github/workflows/version-checks.yml

@@ -19,6 +19,9 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
       - name: Install extra python packages
         run: python3 -m pip install --requirement utils/cq/requirements.txt
       - name: Check ${{ matrix.package }} version

SConstruct

@@ -480,9 +480,8 @@ def scons():
     prereqs.save_build_info()
     # also install to $PREFIX/lib to work with existing avocado test code
     if prereqs.test_requested():
-        env.Install('$PREFIX/lib/daos', ['.build_vars.sh', '.build_vars.json'])
         env.Install('$PREFIX/lib/daos/TESTING/ftest/util', ['site_scons/env_modules.py'])
-        env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh'])
+        env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh', "requirements-ftest.txt"])
 
     env.Install("$PREFIX/lib64/daos", "VERSION")
 

ci/gha_helper.py

@@ -16,6 +16,8 @@
                '.github/workflows/landing-builds.yml',
                '.dockerignore',
                'requirements.txt',
+               'requirements-build.txt',
+               'requirements-utest.txt',
                'ci/gha_helper.py']
 
 COMMIT_CMD = ['git', 'rev-parse', '--short', 'HEAD']

ci/provisioning/post_provision_config_nodes_EL_8.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-#  (C) Copyright 2021-2023 Intel Corporation.
+#  (C) Copyright 2021-2024 Intel Corporation.
 #
 #  SPDX-License-Identifier: BSD-2-Clause-Patent
 
@@ -15,19 +15,12 @@ group_repo_post() {
 }
 
 distro_custom() {
-    # install avocado
-    local avocado_rpms=(python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}})
-    if [ -z "$(dnf repoquery "${avocado_rpms[@]}")" ]; then
-        avocado_rpms=()
-        pip install "avocado-framework<83.0"
-        pip install "avocado-framework-plugin-result-html<83.0"
-        pip install "avocado-framework-plugin-varianter-yaml-to-mux<83.0"
-    fi
-    dnf -y install "${avocado_rpms[@]}" clustershell
-
-    # for Launchable's pip install
-    dnf -y install python3-setuptools.noarch
+    # TODO: This code is not exiting on failure.
 
+    # Use a more recent python version for unit testing, this allows us to also test installing
+    # pydaos into virtual environments.
+    dnf -y install python39 python39-devel
+    dnf -y install python3.11 python3.11-devel
 }
 
 install_mofed() {

ci/provisioning/post_provision_config_nodes_LEAP_15.sh

@@ -16,13 +16,4 @@ distro_custom() {
         sed -e '/MODULEPATH=/s/$/:\/usr\/share\/modules/'                     \
                /etc/profile.d/lmod.sh;                                        \
     fi
-
-    # force install of avocado 69.x
-    dnf -y erase avocado{,-common}                                              \
-                 python2-avocado{,-plugins-{output-html,varianter-yaml-to-mux}}
-    python3 -m pip install --upgrade pip
-    python3 -m pip install "avocado-framework<70.0"
-    python3 -m pip install "avocado-framework-plugin-result-html<70.0"
-    python3 -m pip install "avocado-framework-plugin-varianter-yaml-to-mux<70.0"
-
 }

ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh

@@ -48,9 +48,7 @@ post_provision_config_nodes() {
         fi
     fi
 
-    apt-get -y install avocado python3-avocado-plugins-output-html   \
-                       python3-avocado-plugins-varianter-yaml-to-mux \
-                       lsb-core
+    apt-get -y install lsb-core
 
     # shellcheck disable=2086
     if [ -n "$INST_RPMS" ] &&
@@ -63,10 +61,6 @@ post_provision_config_nodes() {
         return "$rc"
     fi
 
-    # temporary hack until Python 3 is supported by Functional testing
-    # possible TODO: support testing non-RPM testing
-    sed -ie '1s/2/3/' /usr/lib/daos/TESTING/ftest/launch.py
-
     # change the default shell to bash -- we write a lot of bash
     chsh -s /bin/bash
 

ci/rpm/test_daos_node.sh

@@ -101,17 +101,29 @@ sudo chmod 0755 /tmp/daos_sockets
 sudo chown "$me:$me" /tmp/daos_sockets
 
 FTEST=/usr/lib/daos/TESTING/ftest
-sudo PYTHONPATH="$FTEST/util"                               \
-     $FTEST/config_file_gen.py -n "$HOSTNAME"               \
-                               -a /etc/daos/daos_agent.yml  \
-                               -s /etc/daos/daos_server.yml
+
+python3 -m venv venv
+# shellcheck disable=SC1091
+source venv/bin/activate
+pip install --upgrade pip
+pip install -r $FTEST/requirements-ftest.txt
+
+sudo PYTHONPATH="$FTEST/util"                        \
+     "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py -n "$HOSTNAME" \
+        -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml
 sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml'
 sudo PYTHONPATH="$FTEST/util"                        \
-     $FTEST/config_file_gen.py -n "$HOSTNAME"        \
-                               -d /etc/daos/daos_control.yml
+     "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py \
+     -n "$HOSTNAME" -d /etc/daos/daos_control.yml
 cat /etc/daos/daos_server.yml
 cat /etc/daos/daos_agent.yml
 cat /etc/daos/daos_control.yml
+
+# python3.6 does not like deactivate with -u set, later versions are OK with it however.
+set +u
+deactivate
+set -u
+
 if ! module load "$OPENMPI"; then
     echo "Unable to load OpenMPI module: $OPENMPI"
     module avail

ci/unit/required_packages.sh

@@ -2,17 +2,13 @@
 
 set -eux
 
-distro="$1"
 # No longer used but provided by pipeline-lib
+# distro="$1"
 # quick_build="${2:-false}"
 
-if [[ "$distro" = *7 ]]; then
-    OPENMPI_VER="3"
-    PY_MINOR_VER="6"
-elif [[ "$distro" = *8 ]]; then
-    OPENMPI_VER=""
-    PY_MINOR_VER=""
-fi
+OPENMPI_VER=""
+PY_MINOR_VER=""
+
 pkgs="argobots                         \
       boost-python3$PY_MINOR_VER-devel \
       capstone                         \
@@ -32,9 +28,6 @@ pkgs="argobots                         \
       patchelf                         \
       pmix                             \
       protobuf-c                       \
-      python3$PY_MINOR_VER-junit_xml   \
-      python3$PY_MINOR_VER-pyxattr     \
-      python3$PY_MINOR_VER-tabulate    \
       spdk-devel                       \
       valgrind-devel"
 

ci/unit/test_main_node.sh

@@ -73,5 +73,18 @@ fi
 
 rm -rf "$test_log_dir"
 
+# Use default python as that's where storage_estimator is installed.
+python3 -m venv venv
+# shellcheck disable=SC1091
+source venv/bin/activate
+# touch venv/pip.conf
+# pip config set global.progress_bar off
+# pip config set global.no_color true
+
+pip install --upgrade pip
+pip install --requirement requirements-utest.txt
+
+pip install /opt/daos/lib/daos/python/
+
 utils/run_utest.py $RUN_TEST_VALGRIND --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" \
                    $SUDO_ARG

ci/unit/test_nlt.sh

@@ -11,7 +11,7 @@ NODE=${NODELIST%%,*}
 mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
 # Copy over the install tree and some of the build tree.
-rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils jenkins@"$NODE":build/
+rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements-utest.txt jenkins@"$NODE":build/
 
 # shellcheck disable=SC2029
 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "$(cat "$mydir/test_nlt_node.sh")"

ci/unit/test_nlt_node.sh

@@ -24,5 +24,18 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh"
 # ./utils/node_local_test.py --max-log-size ????MiB --dfuse-dir /localhome/jenkins/ \
 #			   --server-valgrind all
 
+# Use the latest version that CI has available.
+python3.11 -m venv venv
+# shellcheck disable=SC1091
+source venv/bin/activate
+touch venv/pip.conf
+pip config set global.progress_bar off
+pip config set global.no_color true
+
+pip install --upgrade pip
+pip install --requirement requirements-utest.txt
+
+pip install /opt/daos/lib/daos/python/
+
 ./utils/node_local_test.py --max-log-size 1700MiB --dfuse-dir /localhome/jenkins/ \
     --log-usage-save nltir.xml --log-usage-export nltr.json all

debian/changelog

@@ -1,3 +1,9 @@
+daos (2.5.101-3) unstable; urgency=medium
+  [ Ashley M. Pittman ]
+  * Updated pydaos install process
+
+ -- Ashley M. Pittman <[email protected]>  Thu, 04 Apr 2024 09:15:00 -0800
+
 daos (2.5.101-2) unstable; urgency=medium
   [ Jan Michalski ]
   * Add dtx_tests to the server-tests package

debian/daos-client-tests.install

@@ -21,6 +21,3 @@ usr/bin/crt_launch
 usr/bin/daos_gen_io_conf
 usr/bin/daos_run_io_conf
 usr/lib64/libdpar.so
-# For avocado tests
-usr/lib/daos/.build_vars.json
-usr/lib/daos/.build_vars.sh

debian/libdaos-dev.dirs

@@ -1,4 +1,5 @@
 usr/lib64
 usr/include
 usr/include/cart
-usr/include/gurt
+usr/include/gurt
+usr/lib/daos/python

debian/libdaos-dev.install

@@ -5,3 +5,4 @@ usr/lib64/libdaos.so
 usr/lib64/libgurt.so
 usr/lib64/libcart.so
 usr/lib64/*.a
+usr/lib/daos/python/*

requirements-build.txt

@@ -0,0 +1,5 @@
+scons
+ninja
+meson
+distro
+pyelftools

requirements-ftest.txt

@@ -0,0 +1,6 @@
+avocado-framework==82
+avocado-framework-plugin-result-html==82
+avocado-framework-plugin-varianter-yaml-to-mux==82
+clustershell
+paramiko
+distro

requirements-utest.txt

@@ -0,0 +1,6 @@
+tabulate
+junit_xml
+pyxattr
+PyYAML
+# Disabled for now because of DAOS-14688
+# pytest

requirements.txt

@@ -1,13 +1,8 @@
-# Packages required to build/test DAOS.
-defusedxml
-distro
-jira
-junit_xml
-meson
-ninja
-pyelftools
-pyxattr
-pyyaml
-scons
-tabulate
-wheel
+# Packages required to build DAOS.
+-r requirements-build.txt
+# Packages required to run ftest.
+-r requirements-ftest.txt
+# Packages required to run unit testing.
+-r requirements-utest.txt
+# Packages used for linting.
+-r utils/cq/requirements.txt