DAOS-15964 test: verify daos_server_helper on server (#15503)

verify daos_server_helper on server instead of the runner/client.
misc cleanup

Skip-unit-tests: true
Skip-fault-injection-test: true

Required-githooks: true

Signed-off-by: Dalton Bohning <[email protected]>

src/tests/ftest/control/daos_server_helper.py

@@ -1,5 +1,5 @@
 """
-  (C) Copyright 2018-2023 Intel Corporation.
+  (C) Copyright 2018-2024 Intel Corporation.
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -8,6 +8,7 @@
 import stat
 
 from apricot import TestWithServers
+from run_utils import run_remote
 from server_utils import ServerFailed
 
 
@@ -34,17 +35,25 @@ def test_daos_server_helper_format(self):
         :avocado: tags=DaosPrivHelperTest,test_daos_server_helper_format
         """
         # Verify that daos_server_helper has the correct permissions
-        self.log.info("Checking daos_server_helper binary permissions")
-        file_stats = os.stat("/usr/bin/daos_server_helper")
+        # Get the result remotely with os.stat so the format is compatible with local code
+        self.log_step("Verify daos_server_helper binary permissions")
+        helper_path = os.path.join(self.bin, "daos_server_helper")
+        cmd = f"python3 -c 'import os; print(os.stat(\"{helper_path}\").st_mode)'"
+        result = run_remote(self.log, self.hostlist_servers, cmd)
+        if not result.passed:
+            self.fail("Failed to get daos_server_helper mode")
+        if not result.homogeneous:
+            self.fail("Non-homogeneous daos_server_helper mode")
+        mode = int(result.joined_stdout)
 
         # regular file, mode 4750
         desired = stat.S_IFREG | stat.S_ISUID | stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
-        actual = file_stats.st_mode & ~stat.S_IRWXO  # mask out Other bits for non-RPM
+        actual = mode & ~stat.S_IRWXO  # mask out Other bits for non-RPM
         if (actual ^ desired) > 0:
-            self.fail("Incorrect daos_server_helper permissions: {}".format(oct(actual)))
+            self.fail(f"Incorrect daos_server_helper permissions: {oct(actual)}")
 
         # Setup server as non-root
-        self.log.info("(0)Preparing to run daos_server as non-root user")
+        self.log_step("Prepare to run daos_server as non-root user")
         self.add_server_manager()
         self.configure_manager(
             "server", self.server_managers[0], self.hostlist_servers, self.hostfile_servers_slots)
@@ -56,31 +65,31 @@ def test_daos_server_helper_format(self):
         # Prep server for format, run command under non-root user
         # Note: This will just report the presence of PMem namespaces if the NVDIMMs are already
         #       configured in AppDirect interleaved mode and namespaces have been created.
-        self.log.info("(1)Performing NVMe storage prepare")
+        self.log_step("Perform NVMe storage prepare as non-root")
         try:
             self.server_managers[0].prepare_storage(user, False, True)
         except ServerFailed as err:
-            self.fail("##(1)Failed preparing NVMe as user {}: {}".format(user, err))
+            self.fail(f"Failed to prepare NVMe as user {user}: {err}")
 
         # Start server
-        self.log.info("(2)Starting server as non-root")
+        self.log_step("Start server as non-root")
         try:
             self.server_managers[0].detect_format_ready()
+            self.register_cleanup(self.stop_servers)
         except ServerFailed as error:
-            self.fail(
-                "##(2)Failed starting server before format as non-root user: {}".format(error))
+            self.fail(f"Failed to start server before format as non-root user: {error}")
 
         # Run format command under non-root user
-        self.log.info("(3)Performing SCM format")
+        self.log_step("Perform SCM format")
         result = self.server_managers[0].dmg.storage_format()
         if result is None:
-            self.fail("##(3)Failed to format storage")
+            self.fail("Failed to format storage")
 
         # Verify format success when all the daos_engine start.
         # Use dmg to detect server start.
-        self.log.info("(4)Verify format success when all the daos_engine start")
+        self.log_step("Verify format succeeds when all the daos_engines start")
         try:
             self.server_managers[0].detect_start_via_dmg = True
             self.server_managers[0].detect_engine_start()
         except ServerFailed as error:
-            self.fail("##(4)Failed starting server after format as non-root user: {}".format(error))
+            self.fail(f"Failed to start server after format as non-root user: {error}")

src/tests/ftest/control/daos_server_helper.yaml

@@ -1,5 +1,3 @@
-# change host names to your reserved nodes, the
-# required quantity is indicated by the placeholders
 hosts:
   test_servers: 1
 timeout: 60