DAOS-15084 tests: fix some CR test scripts issues

1. For test_lost_majority_ps_replicas, remove "rdb-pool" from two ranks that contain the pool service replica. 2. For test_dangling_rank_entry, the rebuild process after CR is not related with CR logic but may cause test timeout, drop it. 3. More log messages when update CR bookmark. 4. More class tags for ddb tests. Signed-off-by: Fan Yong <[email protected]>
daos-stack · Mar 5, 2024 · 48c6d6f · 48c6d6f
1 parent 2b16bc4
commit 48c6d6f
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 38 deletions.
diff --git a/src/chk/chk_vos.c b/src/chk/chk_vos.c
@@ -99,9 +99,9 @@ chk_bk_update_leader(struct chk_bookmark *cbk)
 	int	rc;
 
 	rc = chk_db_update(CHK_BK_LEADER, strlen(CHK_BK_LEADER), cbk, sizeof(*cbk));
-	if (rc != 0)
-		D_ERROR("Failed to update leader bookmark on rank %u: "DF_RC"\n",
-			dss_self_rank(), DP_RC(rc));
+	DL_CDEBUG(rc == 0, DLOG_INFO, DLOG_ERR, rc,
+		  "Update leader bookmark on rank %u, status %u, phase %u",
+		  dss_self_rank(), cbk->cb_ins_status, cbk->cb_phase);
 
 	return rc;
 }
@@ -138,9 +138,9 @@ chk_bk_update_engine(struct chk_bookmark *cbk)
 	int	rc;
 
 	rc = chk_db_update(CHK_BK_ENGINE, strlen(CHK_BK_ENGINE), cbk, sizeof(*cbk));
-	if (rc != 0)
-		D_ERROR("Failed to update engine bookmark on rank %u: "DF_RC"\n",
-			dss_self_rank(), DP_RC(rc));
+	DL_CDEBUG(rc == 0, DLOG_INFO, DLOG_ERR, rc,
+		  "Update engine bookmark on rank %u, status %u, phase %u",
+		  dss_self_rank(), cbk->cb_ins_status, cbk->cb_phase);
 
 	return rc;
 }
@@ -177,9 +177,9 @@ chk_bk_update_pool(struct chk_bookmark *cbk, char *uuid_str)
 	int	rc;
 
 	rc = chk_db_update(uuid_str, strlen(uuid_str), cbk, sizeof(*cbk));
-	if (rc != 0)
-		D_ERROR("Failed to update pool %s bookmark on rank %u: "DF_RC"\n",
-			uuid_str, dss_self_rank(), DP_RC(rc));
+	DL_CDEBUG(rc == 0, DLOG_INFO, DLOG_ERR, rc,
+		  "Update pool %s bookmark on rank %u, status %u, phase %u",
+		  uuid_str, dss_self_rank(), cbk->cb_ins_status, cbk->cb_phase);
 
 	return rc;
 }

diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py
@@ -47,7 +47,7 @@ def test_recovery_ddb_ls(self):
         :avocado: tags=all,full_regression
         :avocado: tags=vm
         :avocado: tags=recovery
-        :avocado: tags=ddb_cmd,test_recovery_ddb_ls
+        :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_ls
         """
         # Create a pool and a container.
         self.add_pool()
@@ -218,7 +218,7 @@ def test_recovery_ddb_rm(self):
         :avocado: tags=all,full_regression
         :avocado: tags=vm
         :avocado: tags=recovery
-        :avocado: tags=ddb_cmd,test_recovery_ddb_rm
+        :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_rm
         """
         # 1. Create a pool and a container. Insert objects, dkeys, and akeys.
         self.add_pool(connect=True)
@@ -365,7 +365,7 @@ def test_recovery_ddb_load(self):
         :avocado: tags=all,full_regression
         :avocado: tags=vm
         :avocado: tags=recovery
-        :avocado: tags=ddb_cmd,test_recovery_ddb_load
+        :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_load
         """
         # 1. Create a pool and a container.
         self.add_pool(connect=True)
@@ -463,7 +463,7 @@ def test_recovery_ddb_dump_value(self):
         :avocado: tags=all,full_regression
         :avocado: tags=vm
         :avocado: tags=recovery
-        :avocado: tags=ddb_cmd,test_recovery_ddb_dump_value
+        :avocado: tags=DdbTest,ddb_cmd,test_recovery_ddb_dump_value
         """
         # 1. Create a pool and a container.
         self.add_pool(connect=True)

diff --git a/src/tests/ftest/recovery/pool_list_consolidation.py b/src/tests/ftest/recovery/pool_list_consolidation.py
@@ -276,13 +276,20 @@ def test_lost_majority_ps_replicas(self):
         dmg_command = self.get_dmg_command()
         dmg_command.system_stop()
 
-        # 3. Remove /mnt/daos/<pool_uuid>/rdb-pool from rank 0 and 2.
-        rank_to_host = self.server_managers[0].ranks
-        host_0 = rank_to_host[0]
-        host_2 = rank_to_host[2]
-        hosts = NodeSet.fromlist([host_0, host_2])
+        # 3. Remove /mnt/daos/<pool_uuid>/rdb-pool from two ranks.
+        rdb_pool_path = f"/mnt/daos0/{self.pool.uuid.lower()}/rdb-pool"
         command = f"sudo rm /mnt/daos0/{self.pool.uuid.lower()}/rdb-pool"
-        pcmd(hosts=hosts, command=command)
+        hosts = list(set(self.server_managers[0].ranks.values()))
+        count = 0
+        for host in hosts:
+            node = NodeSet(host)
+            check_out = check_file_exists(hosts=node, filename=rdb_pool_path, sudo=True)
+            if check_out[0]:
+                pcmd(hosts=node, command=command)
+                self.log.info("rm rdb-pool from %s", str(node))
+                count += 1
+                if count > 1:
+                    break
 
         # 4. Start servers.
         dmg_command.system_start()
@@ -312,10 +319,10 @@ def test_lost_majority_ps_replicas(self):
         # should have rdb-pool.
         hosts = list(set(self.server_managers[0].ranks.values()))
         count = 0
-        rdb_pool_path = f"/mnt/daos0/{self.pool.uuid.lower()}/rdb-pool"
         for host in hosts:
             node = NodeSet(host)
-            if check_file_exists(hosts=node, filename=rdb_pool_path):
+            check_out = check_file_exists(hosts=node, filename=rdb_pool_path, sudo=True)
+            if check_out[0]:
                 count += 1
                 self.log.info("rdb-pool found at %s", str(node))
 

diff --git a/src/tests/ftest/recovery/pool_membership.py b/src/tests/ftest/recovery/pool_membership.py
@@ -314,8 +314,6 @@ def test_dangling_rank_entry(self):
         6. Start checker.
         7. Query the checker until expected number of inconsistencies are repaired.
         8. Disable checker and start servers.
-        9. Wait for rebuild to finish.
-        10. Query the pool and verify that expected number of targets are disabled.
 
         Jira ID: DAOS-11735
 
@@ -391,19 +389,4 @@ def test_dangling_rank_entry(self):
         self.log_step("Disable checker.")
         dmg_command.check_disable()
 
-        # 9. Wait for rebuild to finish.
-        self.log_step("Wait for rebuild to finish.")
-        self.pool.wait_for_rebuild_to_start(interval=5)
-        self.pool.wait_for_rebuild_to_end(interval=5)
-
-        # 10. Query the pool and verify that expected number of targets are disabled.
-        self.log_step(
-            "Query the pool and verify that expected number of targets are disabled.")
-        self.pool.set_query_data()
-        disabled_targets = self.pool.query_data["response"]["disabled_targets"]
-        if disabled_targets != targets:
-            msg = (f"Unexpected number of targets are disabled! Expected = {targets}; "
-                   f"Actual = {disabled_targets}")
-            errors.append(msg)
-
         report_errors(test=self, errors=errors)