DAOS-16229 test: support dynamic stonewall file

support dynamic stonewall file for ior and mdtest by using
DAOS_TEST_LOG_DIR

Features: ior mdtest soak_smoke
Skip-unit-tests: true
Skip-fault-injection-test: true
Allow-unstable-test: true

Required-githooks: true

Signed-off-by: Dalton Bohning <[email protected]>

src/tests/ftest/aggregation/continuous_write.py

@@ -31,7 +31,7 @@ def run_ior_repeat(self, namespace, pool, container):
             pool (TestPool): Pool to use with IOR.
             container (TestContainer): Container to use with IOR.
         """
-        ior_cmd = IorCommand(namespace=namespace)
+        ior_cmd = IorCommand(self.test_env.log_dir, namespace=namespace)
         ior_cmd.get_params(self)
         ior_cmd.set_daos_params(pool, container.identifier)
         testfile = os.path.join(os.sep, "test_file_1")

src/tests/ftest/aggregation/space_rb.py

@@ -43,7 +43,7 @@ def run_ior_verify_error(self, namespace, pool, container, job_num, errors):
             transfer_size (str): Transfer size parameter for the IOR.
             errors (list): List to collect the errors occurred during the test.
         """
-        ior_cmd = IorCommand(namespace=namespace)
+        ior_cmd = IorCommand(self.test_env.log_dir, namespace=namespace)
         ior_cmd.get_params(self)
         ior_cmd.set_daos_params(pool, container.identifier)
         testfile = os.path.join(os.sep, f"test_file_{job_num}")

src/tests/ftest/deployment/agent_failure.py

@@ -32,7 +32,7 @@ def run_ior_collect_error(self, results, job_num, file_name, clients, namespace)
             clients (list): Client hostnames to run IOR from.
             namespace (str): IOR namespace.
         """
-        ior_cmd = IorCommand(namespace=namespace)
+        ior_cmd = IorCommand(self.test_env.log_dir, namespace=namespace)
         ior_cmd.get_params(self)
         ior_cmd.set_daos_params(self.pool, self.container.identifier)
         testfile = os.path.join(os.sep, file_name)

src/tests/ftest/deployment/basic_checkout.yaml

@@ -38,7 +38,7 @@ ior_easy: &ior_easy_base
   block_size: '150G'
   sw_deadline: 30
   sw_wearout: 1
-  sw_status_file: "/var/tmp/daos_testing/stoneWallingStatusFile"
+  sw_status_file: stoneWallingStatusFile
 ior_dfs_sx:
   <<: *ior_easy_base
   api: DFS

src/tests/ftest/deployment/ior_per_rank.yaml

@@ -38,7 +38,7 @@ ior:
   read_flags: "-r -R -C -e -g -G 27 -k -Q 1"
   sw_deadline: 15
   sw_wearout: 1
-  sw_status_file: "/var/tmp/daos_testing/stoneWallingStatusFile"
+  sw_status_file: stoneWallingStatusFile
   dfs_oclass: SX
   transfer_size: 1M
   block_size: 150G

src/tests/ftest/deployment/network_failure.py

@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2022-2023 Intel Corporation.
+  (C) Copyright 2022-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -29,12 +29,12 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.network_down_host = None
         self.interface = None
-        self.test_env = self.params.get("test_environment", "/run/*")
+        self.__test_environment = self.params.get("test_environment", "/run/*")
 
     def pre_tear_down(self):
         """Bring ib0 back up in case the test crashed in the middle."""
         error_list = []
-        if self.test_env == "ci":
+        if self.__test_environment == "ci":
             self.log.debug("Call ip link set before tearDown.")
             if self.network_down_host:
                 update_network_interface(
@@ -60,7 +60,7 @@ def run_ior_report_error(self, results, job_num, file_name, pool, container,
                 infinite.
         """
         # Update the object class depending on the test case.
-        ior_cmd = IorCommand(namespace=namespace)
+        ior_cmd = IorCommand(self.test_env.log_dir, namespace=namespace)
         ior_cmd.get_params(self)
 
         # Standard IOR prep sequence.
@@ -203,7 +203,7 @@ def verify_network_failure(self, ior_namespace, container_namespace):
         self.interface = self.server_managers[0].get_config_value("fabric_iface")
         self.log.info("interface to update = %s", self.interface)
 
-        if self.test_env == "ci":
+        if self.__test_environment == "ci":
             # wolf
             update_network_interface(
                 self.log, interface=self.interface, state="down", hosts=self.network_down_host,
@@ -225,7 +225,7 @@ def verify_network_failure(self, ior_namespace, container_namespace):
         self.log.info(ior_results)
 
         # 4. Bring up the network interface.
-        if self.test_env == "ci":
+        if self.__test_environment == "ci":
             # wolf
             update_network_interface(
                 self.log, interface=self.interface, state="up", hosts=self.network_down_host,
@@ -390,7 +390,7 @@ def test_network_failure_isolation(self):
         self.interface = self.server_managers[0].get_config_value("fabric_iface")
 
         # wolf
-        if self.test_env == "ci":
+        if self.__test_environment == "ci":
             update_network_interface(
                 self.log, interface=self.interface, state="down", hosts=self.network_down_host,
                 errors=errors)
@@ -428,7 +428,7 @@ def test_network_failure_isolation(self):
         self.verify_ior_worked(ior_results=ior_results, job_num=job_num, errors=errors)
 
         # 9. Bring up the network interface.
-        if self.test_env == "ci":
+        if self.__test_environment == "ci":
             # wolf
             update_network_interface(
                 self.log, interface=self.interface, state="up", hosts=self.network_down_host,

src/tests/ftest/deployment/server_rank_failure.py

@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2022-2023 Intel Corporation.
+  (C) Copyright 2022-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -41,7 +41,7 @@ def run_ior_report_error(self, results, job_num, file_name, pool, container,
                 Defaults to None, in which case infinite.
         """
         # Update the object class depending on the test case.
-        ior_cmd = IorCommand(namespace=namespace)
+        ior_cmd = IorCommand(self.test_env.log_dir, namespace=namespace)
         ior_cmd.get_params(self)
 
         # Standard IOR prep sequence.

src/tests/ftest/deployment/target_failure.py

@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2022-2023 Intel Corporation.
+  (C) Copyright 2022-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -36,7 +36,7 @@ def run_ior_report_error(self, results, job_num, file_name, pool, container, nam
             container (TestContainer): Container to run IOR.
         """
         # Update the object class depending on the test case.
-        ior_cmd = IorCommand(namespace=namespace)
+        ior_cmd = IorCommand(self.test_env.log_dir, namespace=namespace)
         ior_cmd.get_params(self)
 
         # Standard IOR prep sequence.

src/tests/ftest/harness/advanced.py

@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2021-2023 Intel Corporation.
+  (C) Copyright 2021-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -102,11 +102,11 @@ def test_launch_failures(self):
         host = NodeSet(choice(self.server_managers[0].hosts))   # nosec
         self.log.info("Creating launch.py failure trigger files on %s", host)
         failure_trigger = "00_trigger-launch-failure_00"
-        failure_trigger_dir = os.path.join(self.base_test_dir, failure_trigger)
+        failure_trigger_dir = os.path.join(self.test_env.log_dir, failure_trigger)
         failure_trigger_files = [
-            os.path.join(self.base_test_dir, "{}_local.yaml".format(failure_trigger)),
+            os.path.join(self.test_env.log_dir, "{}_local.yaml".format(failure_trigger)),
             os.path.join(os.sep, "etc", "daos", "daos_{}.yml".format(failure_trigger)),
-            os.path.join(self.base_test_dir, "{}.log".format(failure_trigger)),
+            os.path.join(self.test_env.log_dir, "{}.log".format(failure_trigger)),
             os.path.join(failure_trigger_dir, "{}.log".format(failure_trigger)),
             os.path.join(os.sep, "tmp", "daos_dump_{}.txt".format(failure_trigger)),
             os.path.join(self.tmp, "valgrind_{}".format(failure_trigger)),

src/tests/ftest/ior/hard.yaml

@@ -59,4 +59,4 @@ ior:
     EC_8P2GX:
       dfs_oclass: "EC_8P2GX"
   sw_wearout: 1
-  sw_status_file: "/var/tmp/daos_testing/stoneWallingStatusFile"
+  sw_status_file: stoneWallingStatusFile

src/tests/ftest/ior/hard_rebuild.yaml

@@ -55,4 +55,4 @@ ior:
       - ["EC_4P2GX", 8]
       - ["EC_8P2GX", 12]
   sw_wearout: 1
-  sw_status_file: "/var/tmp/daos_testing/stoneWallingStatusFile"
+  sw_status_file: stoneWallingStatusFile

src/tests/ftest/nvme/enospace.py

@@ -1,5 +1,5 @@
 '''
-  (C) Copyright 2020-2023 Intel Corporation.
+  (C) Copyright 2020-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 '''
@@ -234,7 +234,7 @@ def ior_bg_thread(self, event):
         self.log.info('----Starting background IOR load----')
 
         # Define the IOR Command and use the parameter from yaml file.
-        ior_bg_cmd = IorCommand()
+        ior_bg_cmd = IorCommand(self.test_env.log_dir)
         ior_bg_cmd.get_params(self)
         ior_bg_cmd.set_daos_params(self.pool, None)
         ior_bg_cmd.dfs_oclass.update(self.ior_cmd.dfs_oclass.value)

src/tests/ftest/nvme/fragmentation.py

@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2020-2023 Intel Corporation.
+  (C) Copyright 2020-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -58,7 +58,7 @@ def ior_runner_thread(self, results):
                                                 self.ior_transfer_size,
                                                 self.ior_flags):
             # Define the arguments for the ior_runner_thread method
-            ior_cmd = IorCommand()
+            ior_cmd = IorCommand(self.test_env.log_dir)
             ior_cmd.get_params(self)
             cont_label = self.label_generator.get_label('cont')
             ior_cmd.set_daos_params(self.pool, cont_label)

src/tests/ftest/nvme/pool_capacity.py

@@ -50,7 +50,7 @@ def ior_thread(self, pool, oclass, api, test, flags, results):
         processes = self.params.get("slots", "/run/ior/clientslots/*")
 
         # Define the arguments for the ior_runner_thread method
-        ior_cmd = IorCommand()
+        ior_cmd = IorCommand(self.test_env.log_dir)
         ior_cmd.get_params(self)
         ior_cmd.set_daos_params(pool, self.label_generator.get_label('TestContainer'))
         ior_cmd.dfs_oclass.update(oclass)

src/tests/ftest/performance/ior_easy.yaml

@@ -44,7 +44,7 @@ ior: &ior_base
   block_size: '150G'
   sw_deadline: 30
   sw_wearout: 1
-  sw_status_file: "/var/tmp/daos_testing/stoneWallingStatusFile"
+  sw_status_file: stoneWallingStatusFile
 
 ior_sx: &ior_sx
   <<: *ior_base

src/tests/ftest/performance/ior_hard.yaml

@@ -46,7 +46,7 @@ ior: &ior_base
   segment_count: 10000000
   sw_deadline: 30
   sw_wearout: 1
-  sw_status_file: "/var/tmp/daos_testing/stoneWallingStatusFile"
+  sw_status_file: stoneWallingStatusFile
 
 ior_sx: &ior_sx
   <<: *ior_base

src/tests/ftest/rebuild/continues_after_stop.py

@@ -26,7 +26,7 @@ def run_ior_basic(self, namespace, pool, container):
             pool (TestPool): Pool to use with IOR.
             container (TestContainer): Container to use with IOR.
         """
-        ior_cmd = IorCommand(namespace=namespace)
+        ior_cmd = IorCommand(self.test_env.log_dir, namespace=namespace)
         ior_cmd.get_params(self)
         ior_cmd.set_daos_params(pool, container.identifier)
         testfile = os.path.join(os.sep, "test_file_1")

src/tests/ftest/server/metadata.py

@@ -481,7 +481,7 @@ def test_metadata_server_restart(self):
             # Create the IOR threads
             for index in range(total_ior_threads):
                 # Define the arguments for the run_ior_loop method
-                ior_cmd = IorCommand()
+                ior_cmd = IorCommand(self.test_env.log_dir)
                 ior_cmd.get_params(self)
                 ior_cmd.set_daos_params(self.pool, None)
                 ior_cmd.flags.value = self.params.get("ior{}flags".format(operation), "/run/ior/*")

src/tests/ftest/util/apricot/apricot/test.py

@@ -68,10 +68,11 @@ def __init__(self, *args, **kwargs):
         self.test_id = self.get_test_name()
 
         # Define a test unique temporary directory
-        self.base_test_dir = os.getenv("DAOS_TEST_LOG_DIR", "/tmp")
-        self.test_dir = os.path.join(self.base_test_dir, self.test_id)
-        if not os.path.exists(self.test_dir):
-            os.makedirs(self.test_dir)
+        self.test_env = TestEnvironment()
+        self.test_dir = os.path.join(self.test_env.log_dir, self.test_id)
+
+        # Create a test unique temporary directory on this host
+        os.makedirs(self.test_dir, exist_ok=True)
 
         # Support unique test case timeout values.  These test case specific
         # timeouts are read from the test yaml using the test case method name

src/tests/ftest/util/ior_test_base.py

@@ -43,7 +43,7 @@ def setUp(self):
         super().setUp()
 
         # Get the parameters for IOR
-        self.ior_cmd = IorCommand()
+        self.ior_cmd = IorCommand(self.test_env.log_dir)
         self.ior_cmd.get_params(self)
         self.processes = self.params.get("np", '/run/ior/client_processes/*')
         self.ppn = self.params.get("ppn", '/run/ior/client_processes/*')

src/tests/ftest/util/ior_utils.py

@@ -9,7 +9,7 @@
 
 from avocado.utils.process import CmdResult
 from command_utils import SubProcessCommand
-from command_utils_base import BasicParameter, FormattedParameter
+from command_utils_base import BasicParameter, FormattedParameter, LogParameter
 from duns_utils import format_path
 from exception_utils import CommandFailure
 from general_utils import get_log_file
@@ -222,7 +222,7 @@ class IorCommand(SubProcessCommand):
 
     Example:
         >>> # Typical use inside of a DAOS avocado test method.
-        >>> ior_cmd = IorCommand()
+        >>> ior_cmd = IorCommand(self.test_env.log_dir)
         >>> ior_cmd.get_params(self)
         >>> ior_cmd.set_daos_params(pool, container)
         >>> mpirun = Mpirun()
@@ -234,13 +234,15 @@ class IorCommand(SubProcessCommand):
         >>> mpirun.run()
     """
 
-    def __init__(self, namespace="/run/ior/*"):
+    def __init__(self, log_dir, namespace="/run/ior/*"):
         """Create an IorCommand object.
 
         Args:
+            log_dir (str): directory in which to put log files
             namespace (str, optional): path to yaml parameters. Defaults to "/run/ior/*".
         """
         super().__init__(namespace, "ior", timeout=60)
+        self._log_dir = log_dir
 
         # Flags
         self.flags = FormattedParameter("{}")
@@ -291,8 +293,7 @@ def __init__(self, namespace="/run/ior/*"):
             "-O stoneWallingWearOut={}")
         self.sw_wearout_iteration = FormattedParameter(
             "-O stoneWallingWearOutIterations={}")
-        self.sw_status_file = FormattedParameter(
-            "-O stoneWallingStatusFile={}")
+        self.sw_status_file = LogParameter(self._log_dir, "-O stoneWallingStatusFile={}", None)
         self.task_offset = FormattedParameter("-Q {}")
         self.segment_count = FormattedParameter("-s {}")
         self.transfer_size = FormattedParameter("-t {}")
@@ -529,7 +530,7 @@ def __init__(self, test, manager, hosts, path=None, slots=None, namespace="/run/
         """
         self.manager = manager
         self.manager.assign_hosts(hosts, path, slots)
-        self.manager.job = IorCommand(namespace)
+        self.manager.job = IorCommand(test.test_env.log_dir, namespace)
         self.manager.job.get_params(test)
         self.manager.output_check = "both"
         self.timeout = test.params.get("timeout", namespace, None)

src/tests/ftest/util/mdtest_test_base.py

@@ -43,7 +43,7 @@ def setUp(self):
         super().setUp()
 
         # Get the parameters for Mdtest
-        self.mdtest_cmd = MdtestCommand()
+        self.mdtest_cmd = MdtestCommand(self.test_env.log_dir)
         self.mdtest_cmd.get_params(self)
         self.ppn = self.params.get("ppn", '/run/mdtest/client_processes/*')
         self.processes = self.params.get("np", '/run/mdtest/client_processes/*')

src/tests/ftest/util/mdtest_utils.py

@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2019-2023 Intel Corporation.
+  (C) Copyright 2019-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -8,16 +8,23 @@
 import re
 
 from command_utils import ExecutableCommand
-from command_utils_base import FormattedParameter
+from command_utils_base import FormattedParameter, LogParameter
 from general_utils import get_log_file
 
 
 class MdtestCommand(ExecutableCommand):
     """Defines a object representing a mdtest command."""
 
-    def __init__(self):
-        """Create an MdtestCommand object."""
+    def __init__(self, log_dir):
+        """Create an MdtestCommand object.
+
+        Args:
+            log_dir (str): directory in which to put log files
+        """
         super().__init__("/run/mdtest/*", "mdtest")
+
+        self._log_dir = log_dir
+
         self.flags = FormattedParameter("{}")   # mdtest flags
         # Optional arguments
         #  -a=STRING             API for I/O [POSIX|DFS|DUMMY]
@@ -63,7 +70,7 @@ def __init__(self):
         self.verbosity_value = FormattedParameter("-V {}")
         self.write_bytes = FormattedParameter("-w {}")
         self.stonewall_timer = FormattedParameter("-W {}")
-        self.stonewall_statusfile = FormattedParameter("-x {}")
+        self.stonewall_statusfile = LogParameter(self._log_dir, "-x {}", None)
         self.depth = FormattedParameter("-z {}")
 
         # Module DFS

src/tests/ftest/util/nvme_utils.py

@@ -131,7 +131,7 @@ def setUp(self):
         # Start the servers and agents
         super().setUp()
         self.hostfile_clients = None
-        self.ior_local_cmd = IorCommand()
+        self.ior_local_cmd = IorCommand(self.test_env.log_dir)
         self.ior_local_cmd.get_params(self)
         self.ior_default_flags = self.ior_local_cmd.flags.value
         self.ior_scm_xfersize = self.params.get("transfer_size",