From e6d1423bbc1da7fcb9217aef0ae164ffa38bd641 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Mon, 1 Aug 2022 17:13:54 +0000 Subject: [PATCH 01/18] DAOS-10830 test: Add cat_recovery/ddb.py to test ddb Test the following ddb subcommands: ls: test_ls() rm: test_rm() load: test_load() dump_value: test_dump_value() See code or the ticket for the test steps. Add container_list_objects() in daos_utils.py Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: ddb Signed-off-by: Makito Kano --- src/tests/ftest/SConscript | 2 +- src/tests/ftest/cat_recovery/ddb.py | 573 ++++++++++++++++++++++++++ src/tests/ftest/cat_recovery/ddb.yaml | 20 + src/tests/ftest/util/daos_utils.py | 19 + 4 files changed, 613 insertions(+), 1 deletion(-) create mode 100644 src/tests/ftest/cat_recovery/ddb.py create mode 100644 src/tests/ftest/cat_recovery/ddb.yaml diff --git a/src/tests/ftest/SConscript b/src/tests/ftest/SConscript index 189033bf27a..9f94dbbcf83 100644 --- a/src/tests/ftest/SConscript +++ b/src/tests/ftest/SConscript @@ -10,7 +10,7 @@ def scons(): env.Install(ftest_install_dir, Glob('*.*')) - dirs = ['aggregation', 'fault_injection', 'checksum', + dirs = ['aggregation', 'fault_injection', 'cat_recovery', 'checksum', 'container', 'control', 'dfuse', 'dtx', 'daos_perf', 'daos_racer', 'daos_vol', 'daos_test', 'data', 'fault_domain', 'io', 'ior', diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py new file mode 100644 index 00000000000..2d75439b3f1 --- /dev/null +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -0,0 +1,573 @@ +""" + (C) Copyright 2022 Intel Corporation. + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os +import ctypes +import re + +from apricot import TestWithServers +from pydaos.raw import IORequest, DaosObjClass +from general_utils import create_string_buffer, report_errors +from ddb_utils import DdbCommand + +SAMPLE_DKEY = "Sample dkey" +SAMPLE_AKEY = "Sample akey" +SAMPLE_DATA = "Sample data" + + +class DdbTest(TestWithServers): + """Test ddb subcommands. + + :avocado: recursive + """ + + def __init__(self, *args, **kwargs): + """Initialize a DdbTest object.""" + super().__init__(*args, **kwargs) + self.ioreqs = [] + self.dkeys = [] + self.akeys = [] + self.data_list = [] + + def insert_objects(self, object_count, dkey_count, akey_count): + """Insert objects, dkeys, akeys, and data into the container. + + Inserted objects: self.ioreqs + Inserted dkeys: self.dkeys + Inserted akeys: self.akeys + Inserted data: self.data_list + + Args: + object_count (int): Number of objects to insert. + dkey_count (int): Number of dkeys to insert. + akey_count (int): Number of akeys to insert. + """ + self.container.open() + + for obj_index in range(object_count): + # Insert object. + self.ioreqs.append(IORequest( + context=self.context, container=self.container.container, obj=None, + objtype=DaosObjClass.OC_S1)) + + for dkey_index in range(dkey_count): + # Prepare the dkey to insert into the object. + dkey_str = " ".join( + [SAMPLE_DKEY, str(obj_index), str(dkey_index)]).encode("utf-8") + self.dkeys.append( + create_string_buffer(value=dkey_str, size=len(dkey_str))) + + for akey_index in range(akey_count): + # Prepare the akey to insert into the dkey. + akey_str = " ".join( + [SAMPLE_AKEY, str(obj_index), str(dkey_index), + str(akey_index)]).encode("utf-8") + self.akeys.append( + create_string_buffer(value=akey_str, size=len(akey_str))) + + # Prepare the data to insert into the akey. + data_str = " ".join( + [SAMPLE_DATA, str(obj_index), str(dkey_index), + str(akey_index)]).encode("utf-8") + self.data_list.append( + create_string_buffer(value=data_str, size=len(data_str))) + c_size = ctypes.c_size_t(ctypes.sizeof(self.data_list[-1])) + + # Insert dkeys, akeys, and the data. + self.ioreqs[-1].single_insert( + dkey=self.dkeys[-1], akey=self.akeys[-1], + value=self.data_list[-1], size=c_size) + + def test_ls(self): + """Test ddb ls. + + 1. Verify container UUID. + 2. Verify object count in the container. + 3. Verify there are two dkeys for every object. Also verify the dkey string and + the size. + 4. Verify there is one akey for every dkey. Also verify the key string and the + size. + 5. Verify data length in every akey. + 6. Restart the server for the cleanup. + 7. Reset the container and the pool to prepare for the cleanup. + + :avocado: tags=all,weekly_regression + :avocado: tags=vm + :avocado: tags=cat_rec + :avocado: tags=ddb_ls + """ + # Create a pool and a container. + self.add_pool() + self.add_container(pool=self.pool) + + ddb_command = DdbCommand( + path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, + vos_file="vos-0") + + errors = [] + + # Insert objects with API. + object_count = self.params.get("object_count", "/run/*") + dkey_count = self.params.get("dkey_count", "/run/*") + akey_count = self.params.get("akey_count", "/run/*") + self.insert_objects( + object_count=object_count, dkey_count=dkey_count, akey_count=akey_count) + + # Need to stop the server to use ddb. + dmg_command = self.get_dmg_command() + dmg_command.system_stop() + + # 1. Verify container UUID. + cmd_result = ddb_command.list_component() + # Sample output. + # [0] d4e0c836-17bd-4df3-b255-929732486bab + match = re.findall( + r"(\[\d+\])\s+([a-f0-9\-]+)", cmd_result.stdout.decode("utf-8")) + self.log.info("List container match = %s", match) + + actual_uuid = match[0][1].lower() + expected_uuid = self.container.uuid.lower() + if actual_uuid != expected_uuid: + msg = "Unexpected container UUID! Expected = {}; Actual = {}".format( + expected_uuid, actual_uuid) + errors.append(msg) + + # 2. Verify object count in the container. + cmd_result = ddb_command.list_component(component_path="[0]") + self.log.debug("## List objects = %s", cmd_result.stdout.decode("utf-8")) + # Sample output. + # /d4e0c836-17bd-4df3-b255-929732486bab/ + # [0] '281479271677953.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) + # [1] '281479271677954.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) + # [2] '281479271677955.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) + # [3] '281479271677956.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) + # [4] '281479271677957.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) + match = re.findall( + r"(\[\d+\])\s+\'(\d+\.\d+)\'", cmd_result.stdout.decode("utf-8")) + self.log.info("List objects match = %s", match) + + actual_object_count = len(match) + expected_object_count = self.params.get("object_count", "/run/*") + if actual_object_count != expected_object_count: + msg = "Unexpected object count! Expected = {}; Actual = {}".format( + expected_object_count, actual_object_count) + errors.append(msg) + + # 3. Verify there are two dkeys for every object. Also verify the dkey string and + # the size. + dkey_akey_regex = r"(\[\d+\])\s+\'(.+)\'\s+\((\d+)\)" + actual_dkey_count = 0 + for obj_index in range(object_count): + component_path = "[0]/[{}]".format(obj_index) + cmd_result = ddb_command.list_component(component_path=component_path) + self.log.debug("## List dkeys %d = %s", obj_index, cmd_result.stdout) + # Sample output. + # /d4e0c836-17bd-4df3-b255-929732486bab/281479271677953.0.0/ + # [0] 'Sample dkey 0 0' (15) + # [1] 'Sample dkey 0 1' (15) + match = re.findall(dkey_akey_regex, cmd_result.stdout.decode("utf-8")) + + actual_dkey_count += len(match) + + # Verify dkey string. + actual_dkey_1 = " ".join(match[0][1].split()) + actual_dkey_2 = " ".join(match[1][1].split()) + # We're not testing the numbers in the string because it's not deterministic. + if SAMPLE_DKEY not in actual_dkey_1: + msg = ("Unexpected dkey! obj_i = {}. Expected = {}; " + "Actual = {}").format(obj_index, SAMPLE_DKEY, actual_dkey_1) + if SAMPLE_DKEY not in actual_dkey_2: + msg = ("Unexpected dkey! obj_i = {}. Expected = {}; " + "Actual = {}").format(obj_index, SAMPLE_DKEY, actual_dkey_2) + + # Verify the dkey size field. + for dkey in match: + dkey_string = dkey[1] + dkey_size = int(dkey[2]) + if len(dkey_string) != dkey_size: + msg = ("Wrong dkey size! obj_index = {}. String = {}; " + "Size = {}").format(obj_index, dkey_string, dkey_size) + errors.append(msg) + + # Verify there are two dkeys for every object. + expected_dkey_count = object_count * dkey_count + if actual_dkey_count != expected_dkey_count: + msg = "Unexpected number of dkeys! Expected = {}; Actual = {}".format( + expected_dkey_count, actual_dkey_count) + errors.append(msg) + + # 4. Verify there is one akey for every dkey. Also verify the key string and the + # size. + akey_count = 0 + for obj_index in range(object_count): + for dkey_index in range(dkey_count): + component_path = "[0]/[{}]/[{}]".format(obj_index, dkey_index) + cmd_result = ddb_command.list_component(component_path=component_path) + msg = "List akeys obj_index = {}, dkey_index = {}, stdout = {}".format( + obj_index, dkey_index, cmd_result.stdout) + self.log.info(msg) + # Output is in the same format as dkey, so use the same regex. + # /d4e0c836-17bd-4df3-b255-929732486bab/281479271677954.0.0/' + # Sample dkey 1 0'/ + # [0] 'Sample akey 1 0 0' (17) + match = re.findall(dkey_akey_regex, cmd_result.stdout.decode("utf-8")) + + akey_count += len(match) + + # Verify akey string. As in dkey, ignore the numbers at the end. + actual_akey = " ".join(match[0][1].split()) + if SAMPLE_AKEY not in actual_akey: + msg = ("Unexpected akey! obj_index = {}; dkey_index = {}; " + "Expected = {}; Actual = {}").format( + obj_index, dkey_index, SAMPLE_AKEY, actual_akey) + errors.append(msg) + + # Verify akey size. + akey_string = match[0][1] + akey_size = int(match[0][2]) + if len(akey_string) != akey_size: + msg = ("Wrong akey size! obj_index = {}; dkey_index = {}; " + "akey = {}; akey size = {}").format( + obj_index, dkey_index, akey_string, akey_size) + errors.append(msg) + + # Verify there is one akey for every dkey. + if expected_dkey_count != akey_count: + msg = "Unexpected number of akeys! Expected = {}; Actual = {}".format( + expected_dkey_count, akey_count) + errors.append(msg) + + # 5. Verify data length in every akey. + for obj_index in range(object_count): + for dkey_index in range(dkey_count): + component_path = "[0]/[{}]/[{}]/[0]".format(obj_index, dkey_index) + cmd_result = ddb_command.list_component(component_path=component_path) + path_stat = ("akey data obj_index = {}, dkey_index = {}, " + "stdout = {}").format( + obj_index, dkey_index, cmd_result.stdout) + self.log.info(path_stat) + # Sample output. + # [0] Single Value (Length: 17 bytes) + match = re.findall( + r"Length:\s+(\d+)\s+bytes", cmd_result.stdout.decode("utf-8")) + data_length = int(match[0]) + expected_data_length = len(SAMPLE_DATA) + 6 + if data_length != expected_data_length: + msg = "Wrong data length! {}; Expected = {}; Actual = {}".format( + path_stat, expected_data_length, data_length) + errors.append(msg) + + # 6. Restart the server for the cleanup. + dmg_command.system_start() + + # 7. Reset the container and the pool to prepare for the cleanup. + self.container.close() + self.pool.disconnect() + self.pool.connect() + self.container.open() + self.get_dmg_command().system_start() + + self.log.info("##### Errors #####") + report_errors(test=self, errors=errors) + self.log.info("##################") + + def test_rm(self): + """Test rm. + + 1. Create a pool and a container. Insert objects, dkeys, and akeys. + 2. Stop the server to use ddb. + 3. Call ddb rm to remove the akey. + 4. Restart the server to use the API. + 5. Reset the object, container, and pool to use the API after server restart. + 6. Call list_akey() in pydaos API to verify that the akey was removed. + 7. Stop the server to use ddb. + 8. Call ddb rm to remove the dkey. + 9. Restart the server to use the API. + 10. Reset the object, container, and pool to use the API after server restart. + 11. Call list_dkey() in pydaos API to verify that the dkey was removed. + 12. Stop the server to use ddb. + 13. Call ddb rm to remove the object. + 14. Restart the server to use daos command. + 15. Reset the container and pool so that cleanup works. + 16. Call "daos container list-objects " to verify that the + object was removed. + + :avocado: tags=all,weekly_regression + :avocado: tags=vm + :avocado: tags=cat_rec + :avocado: tags=ddb_rm + """ + # 1. Create a pool and a container. Insert objects, dkeys, and akeys. + self.add_pool(connect=True) + self.add_container(pool=self.pool) + + # Insert one object with one dkey with API. + self.insert_objects(object_count=1, dkey_count=1, akey_count=2) + + # For debugging/reference, check that the dkey and the akey we just inserted are + # returned from the API. + akeys = self.ioreqs[0].list_akey(dkey=self.dkeys[0]) + self.log.info("akeys from API (before) = %s", akeys) + dkeys = self.ioreqs[0].list_dkey() + self.log.info("dkeys from API (before) = %s", dkeys) + + # For debugging/reference, check that the object was inserted using daos command. + obj_list = self.get_daos_command().container_list_objects( + pool=self.pool.uuid, cont=self.container.uuid) + self.log.info("Object list (before) = %s", obj_list.stdout.decode("utf-8")) + + # 2. Need to stop the server to use ddb. + dmg_command = self.get_dmg_command() + dmg_command.system_stop() + + # 3. Call ddb rm to remove the akey. + ddb_command = DdbCommand( + path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, + vos_file="vos-0") + cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]/[0]") + self.log.info("rm akey stdout = %s", cmd_result.stdout) + + # 4. Restart the server to use the API. + dmg_command.system_start() + + # 5. Reset the object, container, and pool to use the API after server restart. + self.ioreqs[0].obj.close() + self.container.close() + self.pool.disconnect() + self.pool.connect() + self.container.open() + self.ioreqs[0].obj.open() + + # 6. Call list_akey() in pydaos API to verify that the akey was removed. + akeys = self.ioreqs[0].list_akey(dkey=self.dkeys[0]) + self.log.info("akeys from API (after) = %s", akeys) + + errors = [] + expected_len = len(self.akeys) - 1 + actual_len = len(akeys) + if actual_len != expected_len: + msg = ("Unexpected number of akeys after ddb rm! " + "Expected = {}; Actual = {}").format(expected_len, actual_len) + errors.append(msg) + + # 7. Stop the server to use ddb. + dmg_command.system_stop() + + # 8. Call ddb rm to remove the dkey. + cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]") + self.log.info("rm dkey stdout = %s", cmd_result.stdout) + + # 9. Restart the server to use the API. + dmg_command.system_start() + + # 10. Reset the object, container, and pool to use the API after server restart. + self.ioreqs[0].obj.close() + self.container.close() + self.pool.disconnect() + self.pool.connect() + self.container.open() + self.ioreqs[0].obj.open() + + # 11. Call list_dkey() in pydaos API to verify that the dkey was removed. + dkeys = self.ioreqs[0].list_dkey() + self.log.info("dkeys from API (after) = %s", dkeys) + + expected_len = len(self.dkeys) - 1 + actual_len = len(dkeys) + if actual_len != expected_len: + msg = ("Unexpected number of dkeys after ddb rm! " + "Expected = {}; Actual = {}").format(expected_len, actual_len) + errors.append(msg) + + # 12. Stop the server to use ddb. + dmg_command.system_stop() + + # 13. Call ddb rm to remove the object. + cmd_result = ddb_command.remove_component(component_path="[0]/[0]") + self.log.info("rm object stdout = %s", cmd_result.stdout) + + # 14. Restart the server to use daos command. + dmg_command.system_start() + + # 15. Reset the container and pool so that cleanup works. + self.container.close() + self.pool.disconnect() + self.pool.connect() + self.container.open() + + # 16. Call "daos container list-objects " to verify that + # the object was removed. + obj_list = self.get_daos_command().container_list_objects( + pool=self.pool.uuid, cont=self.container.uuid) + self.log.info("Object list (after) = %s", obj_list.stdout.decode("utf-8")) + + expected_len = len(self.ioreqs) - 1 + actual_len = len(obj_list) + if actual_len != expected_len: + msg = ("Unexpected number of objecs after ddb rm! " + "Expected = {}; Actual = {}").format(expected_len, actual_len) + errors.append(msg) + + self.log.info("##### Errors #####") + report_errors(test=self, errors=errors) + self.log.info("##################") + + def test_load(self): + """Test ddb load. + + 1. Create a pool and a container. + 2. Insert one object with one dkey with API. + 3. Stop the server to use ddb. + 4. Load new data into [0]/[0]/[0]/[0] + 5. Restart the server. + 6. Reset the object, container, and pool to use the API. + 7. Verify the data in the akey with single_fetch(). + + :avocado: tags=all,weekly_regression + :avocado: tags=vm + :avocado: tags=cat_rec + :avocado: tags=ddb_load + """ + # 1. Create a pool and a container. + self.add_pool(connect=True) + self.add_container(pool=self.pool) + + # 2. Insert one object with one dkey with API. + self.insert_objects(object_count=1, dkey_count=1, akey_count=1) + + # For debugging/reference, call single_fetch and get the data just inserted. + data_size = len(self.data_list[0]) + # Pass in data_size + 1 to single_fetch to avoid the no-space error. + data_size += 1 + data = self.ioreqs[0].single_fetch( + dkey=self.dkeys[0], akey=self.akeys[0], size=data_size) + self.log.info("data (before) = %s", data.value.decode('utf-8')) + + # 3. Stop the server to use ddb. + dmg_command = self.get_dmg_command() + dmg_command.system_stop() + + ddb_command = DdbCommand( + path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, + vos_file="vos-0") + + # 4. Load new data into [0]/[0]/[0]/[0] + load_file_path = os.path.join(self.test_dir, "new_data.txt") + new_data = "New akey data 0123456789" + with open(load_file_path, "w") as file: + file.write(new_data) + + ddb_command.load(component_path="[0]/[0]/[0]/[0]", load_file_path=load_file_path) + + # 5. Restart the server. + dmg_command.system_start() + + # 6. Reset the object, container, and pool to use the API after server restart. + self.ioreqs[0].obj.close() + self.container.close() + self.pool.disconnect() + self.pool.connect() + self.container.open() + self.ioreqs[0].obj.open() + + # 7. Verify the data in the akey with single_fetch(). + data_size = len(new_data) + data_size += 1 + data = self.ioreqs[0].single_fetch( + dkey=self.dkeys[0], akey=self.akeys[0], size=data_size) + actual_data = data.value.decode('utf-8') + self.log.info("data (after) = %s", actual_data) + + errors = [] + if new_data != actual_data: + msg = "ddb load failed! Expected = {}; Actual = {}".format( + new_data, actual_data) + errors.append(msg) + + self.log.info("##### Errors #####") + report_errors(test=self, errors=errors) + self.log.info("##################") + + def test_dump_value(self): + """Test ddb dump_value. + + 1. Create a pool and a container. + 2. Insert one object with one dkey with API. + 3. Stop the server to use ddb. + 4. Dump the two akeys to files. + 5. Verify the content of the files. + 6. Restart the server for the cleanup. + 7. Reset the object, container, and pool to prepare for the cleanup. + + :avocado: tags=all,weekly_regression + :avocado: tags=vm + :avocado: tags=cat_rec + :avocado: tags=ddb_dump_value + """ + # 1. Create a pool and a container. + self.add_pool(connect=True) + self.add_container(pool=self.pool) + + # 2. Insert one object with one dkey with API. + self.insert_objects(object_count=1, dkey_count=1, akey_count=2) + + # 3. Stop the server to use ddb. + dmg_command = self.get_dmg_command() + dmg_command.system_stop() + + # 4. Dump the two akeys to files. + ddb_command = DdbCommand( + path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, + vos_file="vos-0") + + akey1_file_path = os.path.join(self.test_dir, "akey1.txt") + ddb_command.dump_value( + component_path="[0]/[0]/[0]/[0]", out_file_path=akey1_file_path) + akey2_file_path = os.path.join(self.test_dir, "akey2.txt") + ddb_command.dump_value( + component_path="[0]/[0]/[0]/[1]", out_file_path=akey2_file_path) + + # 5. Verify the content of the files. + actual_akey1_data = None + with open(akey1_file_path, "r") as file: + actual_akey1_data = file.readlines()[0] + actual_akey2_data = None + with open(akey2_file_path, "r") as file: + actual_akey2_data = file.readlines()[0] + + errors = [] + str_data_list = [] + # Convert the data to string. + for data in self.data_list: + str_data_list.append(data.value.decode("utf-8")) + # Verify that we were able to obtain the data and akey1 and akey2 aren't the same. + if actual_akey1_data is None or actual_akey2_data is None or \ + actual_akey1_data == actual_akey2_data: + msg = ("Invalid dumped value! Dumped akey1 data = {}; " + "Dumped akey2 data = {}").format(actual_akey1_data, actual_akey2_data) + errors.append(msg) + # Verify that the data we obtained with ddb are the ones we wrote. The order isn't + # deterministic, so check with "in". + if actual_akey1_data not in str_data_list or \ + actual_akey2_data not in str_data_list: + msg = ("Unexpected dumped value! Dumped akey data 1 = {}; " + "Dumped akey data 2 = {}; Expected data list = {}").format( + actual_akey1_data, actual_akey2_data, str_data_list) + errors.append(msg) + + # 6. Restart the server for the cleanup. + dmg_command.system_start() + + # 7. Reset the object, container, and pool to prepare for the cleanup. + self.ioreqs[0].obj.close() + self.container.close() + self.pool.disconnect() + self.pool.connect() + self.container.open() + self.ioreqs[0].obj.open() + + self.log.info("##### Errors #####") + report_errors(test=self, errors=errors) + self.log.info("##################") diff --git a/src/tests/ftest/cat_recovery/ddb.yaml b/src/tests/ftest/cat_recovery/ddb.yaml new file mode 100644 index 00000000000..92a047e15e8 --- /dev/null +++ b/src/tests/ftest/cat_recovery/ddb.yaml @@ -0,0 +1,20 @@ +hosts: + test_servers: 1 + +timeout: 1800 + +server_config: + name: daos_server + servers: + targets: 1 + +pool: + control_method: dmg + scm_size: 1G + +container: + control_method: API + +object_count: 5 +dkey_count: 2 +akey_count: 1 diff --git a/src/tests/ftest/util/daos_utils.py b/src/tests/ftest/util/daos_utils.py index 0c56c7ad2d0..c5427ce5115 100644 --- a/src/tests/ftest/util/daos_utils.py +++ b/src/tests/ftest/util/daos_utils.py @@ -598,6 +598,25 @@ def container_list_attrs(self, pool, cont, sys_name=None, verbose=False): ("container", "list-attrs"), pool=pool, cont=cont, sys_name=sys_name, verbose=verbose) + def container_list_objects(self, pool, cont, sys_name=None): + """Call daos container list-objects. + + Args: + pool (str): Pool UUID. + cont (str): Container UUID. + sys_name (str, optional): DAOS system name context for servers. + Defaults to None. + + Returns: + dict: the daos json command output converted to a python dictionary + + Raises: + CommandFailure: if the daos container list-objects command fails. + + """ + return self._get_json_result( + ("container", "list-objects"), pool=pool, cont=cont, sys_name=sys_name) + def container_create_snap(self, pool, cont, snap_name=None, epoch=None, sys_name=None): """Call daos container create-snap. From 73e7c7211c4c77d822c1237a1880e34a02a32348 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Mon, 1 Aug 2022 17:18:25 +0000 Subject: [PATCH 02/18] DAOS-10830 test: Update tag Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: ddb Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 2d75439b3f1..3d9508f02c7 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -96,7 +96,7 @@ def test_ls(self): :avocado: tags=all,weekly_regression :avocado: tags=vm :avocado: tags=cat_rec - :avocado: tags=ddb_ls + :avocado: tags=ddb,ddb_ls """ # Create a pool and a container. self.add_pool() @@ -297,7 +297,7 @@ def test_rm(self): :avocado: tags=all,weekly_regression :avocado: tags=vm :avocado: tags=cat_rec - :avocado: tags=ddb_rm + :avocado: tags=ddb,ddb_rm """ # 1. Create a pool and a container. Insert objects, dkeys, and akeys. self.add_pool(connect=True) @@ -428,7 +428,7 @@ def test_load(self): :avocado: tags=all,weekly_regression :avocado: tags=vm :avocado: tags=cat_rec - :avocado: tags=ddb_load + :avocado: tags=ddb,ddb_load """ # 1. Create a pool and a container. self.add_pool(connect=True) @@ -504,7 +504,7 @@ def test_dump_value(self): :avocado: tags=all,weekly_regression :avocado: tags=vm :avocado: tags=cat_rec - :avocado: tags=ddb_dump_value + :avocado: tags=ddb,ddb_dump_value """ # 1. Create a pool and a container. self.add_pool(connect=True) From 659352308d9a724387648d52967e4ba9597ab1aa Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Mon, 1 Aug 2022 17:28:10 +0000 Subject: [PATCH 03/18] DAOS-10830 test: Reduce local variables and fix typo Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: ddb Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 3d9508f02c7..56fbdcfb36b 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -116,8 +116,7 @@ def test_ls(self): object_count=object_count, dkey_count=dkey_count, akey_count=akey_count) # Need to stop the server to use ddb. - dmg_command = self.get_dmg_command() - dmg_command.system_stop() + self.get_dmg_command().system_stop() # 1. Verify container UUID. cmd_result = ddb_command.list_component() @@ -260,7 +259,7 @@ def test_ls(self): errors.append(msg) # 6. Restart the server for the cleanup. - dmg_command.system_start() + self.get_dmg_command().system_start() # 7. Reset the container and the pool to prepare for the cleanup. self.container.close() @@ -406,7 +405,7 @@ def test_rm(self): expected_len = len(self.ioreqs) - 1 actual_len = len(obj_list) if actual_len != expected_len: - msg = ("Unexpected number of objecs after ddb rm! " + msg = ("Unexpected number of objects after ddb rm! " "Expected = {}; Actual = {}").format(expected_len, actual_len) errors.append(msg) From aba86f14c91ba271b808aba6781edb2cc2b369dc Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Mon, 1 Aug 2022 18:32:59 +0000 Subject: [PATCH 04/18] DAOS-10830 test: Fixed pylint Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: ddb Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 56fbdcfb36b..504427c3757 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -543,14 +543,14 @@ def test_dump_value(self): str_data_list.append(data.value.decode("utf-8")) # Verify that we were able to obtain the data and akey1 and akey2 aren't the same. if actual_akey1_data is None or actual_akey2_data is None or \ - actual_akey1_data == actual_akey2_data: + actual_akey1_data == actual_akey2_data: msg = ("Invalid dumped value! Dumped akey1 data = {}; " "Dumped akey2 data = {}").format(actual_akey1_data, actual_akey2_data) errors.append(msg) # Verify that the data we obtained with ddb are the ones we wrote. The order isn't # deterministic, so check with "in". if actual_akey1_data not in str_data_list or \ - actual_akey2_data not in str_data_list: + actual_akey2_data not in str_data_list: msg = ("Unexpected dumped value! Dumped akey data 1 = {}; " "Dumped akey data 2 = {}; Expected data list = {}").format( actual_akey1_data, actual_akey2_data, str_data_list) From ef16a350a9d75040ee134c1bca2f8ba42bbcbbe0 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 3 Aug 2022 12:09:22 +0000 Subject: [PATCH 05/18] DAOS-10830 test: Obtain vos file name from the pool mount point Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: ddb_load Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 38 +++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 504427c3757..9df220e390c 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -6,10 +6,11 @@ import os import ctypes import re +from ClusterShell.NodeSet import NodeSet from apricot import TestWithServers from pydaos.raw import IORequest, DaosObjClass -from general_utils import create_string_buffer, report_errors +from general_utils import create_string_buffer, report_errors, run_pcmd from ddb_utils import DdbCommand SAMPLE_DKEY = "Sample dkey" @@ -419,10 +420,11 @@ def test_load(self): 1. Create a pool and a container. 2. Insert one object with one dkey with API. 3. Stop the server to use ddb. - 4. Load new data into [0]/[0]/[0]/[0] - 5. Restart the server. - 6. Reset the object, container, and pool to use the API. - 7. Verify the data in the akey with single_fetch(). + 4. Find the vos file name. e.g., vos-0. + 5. Load new data into [0]/[0]/[0]/[0] + 6. Restart the server. + 7. Reset the object, container, and pool to use the API. + 8. Verify the data in the akey with single_fetch(). :avocado: tags=all,weekly_regression :avocado: tags=vm @@ -448,11 +450,27 @@ def test_load(self): dmg_command = self.get_dmg_command() dmg_command.system_stop() + # 4. Find the vos file name. + hosts = NodeSet(self.hostlist_servers[0]) + vos_path = os.path.join("/mnt/daos", self.pool.uuid.lower()) + command = " ".join(["sudo", "ls", vos_path]) + cmd_out = run_pcmd(hosts=hosts, command=command) + self.log.debug("## sudo ls /mnt/daos/ output = %s", cmd_out[0]["stdout"]) + + vos_file = None + for file in cmd_out[0]["stdout"]: + if "vos" in file: + vos_file = file + break + if not vos_file: + self.fail("vos file wasn't found in /mnt/daos/%s", self.pool.uuid.lower()) + else: + self.log.debug("## vos_file: %s", vos_file) ddb_command = DdbCommand( path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, - vos_file="vos-0") + vos_file=vos_file) - # 4. Load new data into [0]/[0]/[0]/[0] + # 5. Load new data into [0]/[0]/[0]/[0] load_file_path = os.path.join(self.test_dir, "new_data.txt") new_data = "New akey data 0123456789" with open(load_file_path, "w") as file: @@ -460,10 +478,10 @@ def test_load(self): ddb_command.load(component_path="[0]/[0]/[0]/[0]", load_file_path=load_file_path) - # 5. Restart the server. + # 6. Restart the server. dmg_command.system_start() - # 6. Reset the object, container, and pool to use the API after server restart. + # 7. Reset the object, container, and pool to use the API after server restart. self.ioreqs[0].obj.close() self.container.close() self.pool.disconnect() @@ -471,7 +489,7 @@ def test_load(self): self.container.open() self.ioreqs[0].obj.open() - # 7. Verify the data in the akey with single_fetch(). + # 8. Verify the data in the akey with single_fetch(). data_size = len(new_data) data_size += 1 data = self.ioreqs[0].single_fetch( From 66fd37643e8336eb131183b3ff7f5393cc282e78 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 3 Aug 2022 12:13:54 +0000 Subject: [PATCH 06/18] DAOS-10830 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: ddb_load Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 9df220e390c..63fbac086c0 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -463,7 +463,8 @@ def test_load(self): vos_file = file break if not vos_file: - self.fail("vos file wasn't found in /mnt/daos/%s", self.pool.uuid.lower()) + self.fail( + "vos file wasn't found in /mnt/daos/{}".format(self.pool.uuid.lower())) else: self.log.debug("## vos_file: %s", vos_file) ddb_command = DdbCommand( From 3af54233ac19caed690bb3dd4e496ec67f03355a Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 3 Aug 2022 12:56:22 +0000 Subject: [PATCH 07/18] DAOS-10830 test: Update comment Skip-unit-tests: true Skip-fault-injection-test: true Skip-build: true Test-tag: ddb_load Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 63fbac086c0..e59a48aded9 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -418,7 +418,7 @@ def test_load(self): """Test ddb load. 1. Create a pool and a container. - 2. Insert one object with one dkey with API. + 2. Insert one object with one dkey with the API. 3. Stop the server to use ddb. 4. Find the vos file name. e.g., vos-0. 5. Load new data into [0]/[0]/[0]/[0] From 5a9e279481e2e640c40c4bf6246f39929350ed7f Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 3 Aug 2022 13:07:07 +0000 Subject: [PATCH 08/18] DAOS-10830 test: Update comment Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb_load Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index e59a48aded9..7f8495795fd 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -420,7 +420,7 @@ def test_load(self): 1. Create a pool and a container. 2. Insert one object with one dkey with the API. 3. Stop the server to use ddb. - 4. Find the vos file name. e.g., vos-0. + 4. Find the vos file name. e.g., /mnt/daos//vos-0. 5. Load new data into [0]/[0]/[0]/[0] 6. Restart the server. 7. Reset the object, container, and pool to use the API. From bddbfbbfcc0f43ee5ee8d8689c67b98396bfe966 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 3 Aug 2022 18:38:02 +0000 Subject: [PATCH 09/18] DAOS-10830 test: Add get_vos_file_path() Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb_load ddb_ls Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 50 +++++++++++++++++++---------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 7f8495795fd..c6a5ad2bebc 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -81,6 +81,36 @@ def insert_objects(self, object_count, dkey_count, akey_count): dkey=self.dkeys[-1], akey=self.akeys[-1], value=self.data_list[-1], size=c_size) + def get_vos_file_path(self): + """Get the VOS file path. + + Returns: + str: VOS file path such as /mnt/daos//vos-0 + + """ + hosts = NodeSet(self.hostlist_servers[0]) + vos_path = os.path.join("/mnt/daos", self.pool.uuid.lower()) + command = " ".join(["sudo", "ls", vos_path]) + cmd_out = run_pcmd(hosts=hosts, command=command) + self.log.debug( + "## sudo ls /mnt/daos/%s output = %s", self.pool.uuid.lower(), + cmd_out[0]["stdout"]) + + vos_file = None + for file in cmd_out[0]["stdout"]: + # Assume the VOS file has "vos" in the file name. + if "vos" in file: + vos_file = file + break + + if not vos_file: + self.fail( + "vos file wasn't found in /mnt/daos/{}".format(self.pool.uuid.lower())) + else: + self.log.debug("## vos_file: %s", vos_file) + + return vos_file + def test_ls(self): """Test ddb ls. @@ -103,9 +133,10 @@ def test_ls(self): self.add_pool() self.add_container(pool=self.pool) + vos_file = self.get_vos_file_path() ddb_command = DdbCommand( path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, - vos_file="vos-0") + vos_file=vos_file) errors = [] @@ -451,22 +482,7 @@ def test_load(self): dmg_command.system_stop() # 4. Find the vos file name. - hosts = NodeSet(self.hostlist_servers[0]) - vos_path = os.path.join("/mnt/daos", self.pool.uuid.lower()) - command = " ".join(["sudo", "ls", vos_path]) - cmd_out = run_pcmd(hosts=hosts, command=command) - self.log.debug("## sudo ls /mnt/daos/ output = %s", cmd_out[0]["stdout"]) - - vos_file = None - for file in cmd_out[0]["stdout"]: - if "vos" in file: - vos_file = file - break - if not vos_file: - self.fail( - "vos file wasn't found in /mnt/daos/{}".format(self.pool.uuid.lower())) - else: - self.log.debug("## vos_file: %s", vos_file) + vos_file = self.get_vos_file_path() ddb_command = DdbCommand( path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, vos_file=vos_file) From 3236c8c82da9bcf801bf8a137ffc204392a88927 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Fri, 19 Aug 2022 19:24:34 +0000 Subject: [PATCH 10/18] DAOS-10830 test: Pass in server hostname to DdbCommand Run the ddb command on the given server host. Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 152 +++++++++++++++------------- 1 file changed, 83 insertions(+), 69 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index c6a5ad2bebc..012e1d1adde 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -92,9 +92,6 @@ def get_vos_file_path(self): vos_path = os.path.join("/mnt/daos", self.pool.uuid.lower()) command = " ".join(["sudo", "ls", vos_path]) cmd_out = run_pcmd(hosts=hosts, command=command) - self.log.debug( - "## sudo ls /mnt/daos/%s output = %s", self.pool.uuid.lower(), - cmd_out[0]["stdout"]) vos_file = None for file in cmd_out[0]["stdout"]: @@ -107,7 +104,7 @@ def get_vos_file_path(self): self.fail( "vos file wasn't found in /mnt/daos/{}".format(self.pool.uuid.lower())) else: - self.log.debug("## vos_file: %s", vos_file) + self.log.info("vos_file: %s", vos_file) return vos_file @@ -133,10 +130,12 @@ def test_ls(self): self.add_pool() self.add_container(pool=self.pool) + # Find the vos file name. e.g., /mnt/daos//vos-0. vos_file = self.get_vos_file_path() + host = NodeSet(self.hostlist_servers[0]) ddb_command = DdbCommand( - path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, - vos_file=vos_file) + server_host=host, path=self.bin, mount_point="/mnt/daos", + pool_uuid=self.pool.uuid, vos_file=vos_file) errors = [] @@ -154,8 +153,10 @@ def test_ls(self): cmd_result = ddb_command.list_component() # Sample output. # [0] d4e0c836-17bd-4df3-b255-929732486bab - match = re.findall( - r"(\[\d+\])\s+([a-f0-9\-]+)", cmd_result.stdout.decode("utf-8")) + # stdout is a list which contains each line as separate element. Concatenate them + # to single string so that we can apply regex. + ls_out = "\n".join(cmd_result[0]["stdout"]) + match = re.findall(r"(\[\d+\])\s+([a-f0-9\-]+)", ls_out) self.log.info("List container match = %s", match) actual_uuid = match[0][1].lower() @@ -167,7 +168,6 @@ def test_ls(self): # 2. Verify object count in the container. cmd_result = ddb_command.list_component(component_path="[0]") - self.log.debug("## List objects = %s", cmd_result.stdout.decode("utf-8")) # Sample output. # /d4e0c836-17bd-4df3-b255-929732486bab/ # [0] '281479271677953.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) @@ -175,8 +175,8 @@ def test_ls(self): # [2] '281479271677955.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) # [3] '281479271677956.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) # [4] '281479271677957.0' (type: DAOS_OT_MULTI_HASHED, groups: 1) - match = re.findall( - r"(\[\d+\])\s+\'(\d+\.\d+)\'", cmd_result.stdout.decode("utf-8")) + ls_out = "\n".join(cmd_result[0]["stdout"]) + match = re.findall(r"(\[\d+\])\s+\'(\d+\.\d+)\'", ls_out) self.log.info("List objects match = %s", match) actual_object_count = len(match) @@ -193,12 +193,12 @@ def test_ls(self): for obj_index in range(object_count): component_path = "[0]/[{}]".format(obj_index) cmd_result = ddb_command.list_component(component_path=component_path) - self.log.debug("## List dkeys %d = %s", obj_index, cmd_result.stdout) + ls_out = "\n".join(cmd_result[0]["stdout"]) # Sample output. # /d4e0c836-17bd-4df3-b255-929732486bab/281479271677953.0.0/ # [0] 'Sample dkey 0 0' (15) # [1] 'Sample dkey 0 1' (15) - match = re.findall(dkey_akey_regex, cmd_result.stdout.decode("utf-8")) + match = re.findall(dkey_akey_regex, ls_out) actual_dkey_count += len(match) @@ -236,14 +236,15 @@ def test_ls(self): for dkey_index in range(dkey_count): component_path = "[0]/[{}]/[{}]".format(obj_index, dkey_index) cmd_result = ddb_command.list_component(component_path=component_path) + ls_out = "\n".join(cmd_result[0]["stdout"]) msg = "List akeys obj_index = {}, dkey_index = {}, stdout = {}".format( - obj_index, dkey_index, cmd_result.stdout) + obj_index, dkey_index, ls_out) self.log.info(msg) # Output is in the same format as dkey, so use the same regex. # /d4e0c836-17bd-4df3-b255-929732486bab/281479271677954.0.0/' # Sample dkey 1 0'/ # [0] 'Sample akey 1 0 0' (17) - match = re.findall(dkey_akey_regex, cmd_result.stdout.decode("utf-8")) + match = re.findall(dkey_akey_regex, ls_out) akey_count += len(match) @@ -275,14 +276,13 @@ def test_ls(self): for dkey_index in range(dkey_count): component_path = "[0]/[{}]/[{}]/[0]".format(obj_index, dkey_index) cmd_result = ddb_command.list_component(component_path=component_path) + ls_out = "\n".join(cmd_result[0]["stdout"]) path_stat = ("akey data obj_index = {}, dkey_index = {}, " - "stdout = {}").format( - obj_index, dkey_index, cmd_result.stdout) + "stdout = {}").format(obj_index, dkey_index, ls_out) self.log.info(path_stat) # Sample output. # [0] Single Value (Length: 17 bytes) - match = re.findall( - r"Length:\s+(\d+)\s+bytes", cmd_result.stdout.decode("utf-8")) + match = re.findall(r"Length:\s+(\d+)\s+bytes", ls_out) data_length = int(match[0]) expected_data_length = len(SAMPLE_DATA) + 6 if data_length != expected_data_length: @@ -309,20 +309,21 @@ def test_rm(self): 1. Create a pool and a container. Insert objects, dkeys, and akeys. 2. Stop the server to use ddb. - 3. Call ddb rm to remove the akey. - 4. Restart the server to use the API. - 5. Reset the object, container, and pool to use the API after server restart. - 6. Call list_akey() in pydaos API to verify that the akey was removed. - 7. Stop the server to use ddb. - 8. Call ddb rm to remove the dkey. - 9. Restart the server to use the API. - 10. Reset the object, container, and pool to use the API after server restart. - 11. Call list_dkey() in pydaos API to verify that the dkey was removed. - 12. Stop the server to use ddb. - 13. Call ddb rm to remove the object. - 14. Restart the server to use daos command. - 15. Reset the container and pool so that cleanup works. - 16. Call "daos container list-objects " to verify that the + 3. Find the vos file name. e.g., /mnt/daos//vos-0. + 4. Call ddb rm to remove the akey. + 5. Restart the server to use the API. + 6. Reset the object, container, and pool to use the API after server restart. + 7. Call list_akey() in pydaos API to verify that the akey was removed. + 8. Stop the server to use ddb. + 9. Call ddb rm to remove the dkey. + 10. Restart the server to use the API. + 11. Reset the object, container, and pool to use the API after server restart. + 12. Call list_dkey() in pydaos API to verify that the dkey was removed. + 13. Stop the server to use ddb. + 14. Call ddb rm to remove the object. + 15. Restart the server to use daos command. + 16. Reset the container and pool so that cleanup works. + 17. Call "daos container list-objects " to verify that the object was removed. :avocado: tags=all,weekly_regression @@ -345,25 +346,29 @@ def test_rm(self): self.log.info("dkeys from API (before) = %s", dkeys) # For debugging/reference, check that the object was inserted using daos command. - obj_list = self.get_daos_command().container_list_objects( + list_obj_out = self.get_daos_command().container_list_objects( pool=self.pool.uuid, cont=self.container.uuid) - self.log.info("Object list (before) = %s", obj_list.stdout.decode("utf-8")) + self.log.info("Object list (before) = %s", list_obj_out["response"]) # 2. Need to stop the server to use ddb. dmg_command = self.get_dmg_command() dmg_command.system_stop() - # 3. Call ddb rm to remove the akey. + # 3. Find the vos file name. + vos_file = self.get_vos_file_path() + host = NodeSet(self.hostlist_servers[0]) ddb_command = DdbCommand( - path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, - vos_file="vos-0") + server_host=host, path=self.bin, mount_point="/mnt/daos", + pool_uuid=self.pool.uuid, vos_file=vos_file) + + # 4. Call ddb rm to remove the akey. cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]/[0]") - self.log.info("rm akey stdout = %s", cmd_result.stdout) + self.log.info("rm akey stdout = %s", cmd_result[0]["stdout"]) - # 4. Restart the server to use the API. + # 5. Restart the server to use the API. dmg_command.system_start() - # 5. Reset the object, container, and pool to use the API after server restart. + # 6. Reset the object, container, and pool to use the API after server restart. self.ioreqs[0].obj.close() self.container.close() self.pool.disconnect() @@ -371,7 +376,7 @@ def test_rm(self): self.container.open() self.ioreqs[0].obj.open() - # 6. Call list_akey() in pydaos API to verify that the akey was removed. + # 7. Call list_akey() in pydaos API to verify that the akey was removed. akeys = self.ioreqs[0].list_akey(dkey=self.dkeys[0]) self.log.info("akeys from API (after) = %s", akeys) @@ -383,17 +388,17 @@ def test_rm(self): "Expected = {}; Actual = {}").format(expected_len, actual_len) errors.append(msg) - # 7. Stop the server to use ddb. + # 8. Stop the server to use ddb. dmg_command.system_stop() - # 8. Call ddb rm to remove the dkey. + # 9. Call ddb rm to remove the dkey. cmd_result = ddb_command.remove_component(component_path="[0]/[0]/[0]") - self.log.info("rm dkey stdout = %s", cmd_result.stdout) + self.log.info("rm dkey stdout = %s", cmd_result[0]["stdout"]) - # 9. Restart the server to use the API. + # 10. Restart the server to use the API. dmg_command.system_start() - # 10. Reset the object, container, and pool to use the API after server restart. + # 11. Reset the object, container, and pool to use the API after server restart. self.ioreqs[0].obj.close() self.container.close() self.pool.disconnect() @@ -401,7 +406,7 @@ def test_rm(self): self.container.open() self.ioreqs[0].obj.open() - # 11. Call list_dkey() in pydaos API to verify that the dkey was removed. + # 12. Call list_dkey() in pydaos API to verify that the dkey was removed. dkeys = self.ioreqs[0].list_dkey() self.log.info("dkeys from API (after) = %s", dkeys) @@ -412,30 +417,34 @@ def test_rm(self): "Expected = {}; Actual = {}").format(expected_len, actual_len) errors.append(msg) - # 12. Stop the server to use ddb. + # 13. Stop the server to use ddb. dmg_command.system_stop() - # 13. Call ddb rm to remove the object. + # 14. Call ddb rm to remove the object. cmd_result = ddb_command.remove_component(component_path="[0]/[0]") - self.log.info("rm object stdout = %s", cmd_result.stdout) + self.log.info("rm object stdout = %s", cmd_result[0]["stdout"]) - # 14. Restart the server to use daos command. + # 15. Restart the server to use daos command. dmg_command.system_start() - # 15. Reset the container and pool so that cleanup works. + # 16. Reset the container and pool so that cleanup works. self.container.close() self.pool.disconnect() self.pool.connect() self.container.open() - # 16. Call "daos container list-objects " to verify that + # 17. Call "daos container list-objects " to verify that # the object was removed. - obj_list = self.get_daos_command().container_list_objects( + list_obj_out = self.get_daos_command().container_list_objects( pool=self.pool.uuid, cont=self.container.uuid) - self.log.info("Object list (after) = %s", obj_list.stdout.decode("utf-8")) + obj_list = list_obj_out["response"] + self.log.info("Object list (after) = %s", obj_list) expected_len = len(self.ioreqs) - 1 - actual_len = len(obj_list) + if obj_list: + actual_len = len(obj_list) + else: + actual_len = 0 if actual_len != expected_len: msg = ("Unexpected number of objects after ddb rm! " "Expected = {}; Actual = {}").format(expected_len, actual_len) @@ -483,9 +492,10 @@ def test_load(self): # 4. Find the vos file name. vos_file = self.get_vos_file_path() + host = NodeSet(self.hostlist_servers[0]) ddb_command = DdbCommand( - path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, - vos_file=vos_file) + server_host=host, path=self.bin, mount_point="/mnt/daos", + pool_uuid=self.pool.uuid, vos_file=vos_file) # 5. Load new data into [0]/[0]/[0]/[0] load_file_path = os.path.join(self.test_dir, "new_data.txt") @@ -530,10 +540,11 @@ def test_dump_value(self): 1. Create a pool and a container. 2. Insert one object with one dkey with API. 3. Stop the server to use ddb. - 4. Dump the two akeys to files. - 5. Verify the content of the files. - 6. Restart the server for the cleanup. - 7. Reset the object, container, and pool to prepare for the cleanup. + 4. Find the vos file name. e.g., /mnt/daos//vos-0. + 5. Dump the two akeys to files. + 6. Verify the content of the files. + 7. Restart the server for the cleanup. + 8. Reset the object, container, and pool to prepare for the cleanup. :avocado: tags=all,weekly_regression :avocado: tags=vm @@ -551,11 +562,14 @@ def test_dump_value(self): dmg_command = self.get_dmg_command() dmg_command.system_stop() - # 4. Dump the two akeys to files. + # 4. Find the vos file name. + vos_file = self.get_vos_file_path() + host = NodeSet(self.hostlist_servers[0]) ddb_command = DdbCommand( - path=self.bin, mount_point="/mnt/daos", pool_uuid=self.pool.uuid, - vos_file="vos-0") + server_host=host, path=self.bin, mount_point="/mnt/daos", + pool_uuid=self.pool.uuid, vos_file=vos_file) + # 5. Dump the two akeys to files. akey1_file_path = os.path.join(self.test_dir, "akey1.txt") ddb_command.dump_value( component_path="[0]/[0]/[0]/[0]", out_file_path=akey1_file_path) @@ -563,7 +577,7 @@ def test_dump_value(self): ddb_command.dump_value( component_path="[0]/[0]/[0]/[1]", out_file_path=akey2_file_path) - # 5. Verify the content of the files. + # 6. Verify the content of the files. actual_akey1_data = None with open(akey1_file_path, "r") as file: actual_akey1_data = file.readlines()[0] @@ -591,10 +605,10 @@ def test_dump_value(self): actual_akey1_data, actual_akey2_data, str_data_list) errors.append(msg) - # 6. Restart the server for the cleanup. + # 7. Restart the server for the cleanup. dmg_command.system_start() - # 7. Reset the object, container, and pool to prepare for the cleanup. + # 8. Reset the object, container, and pool to prepare for the cleanup. self.ioreqs[0].obj.close() self.container.close() self.pool.disconnect() From 45e51ce3fbc3e669695098d88cf2e892a04025dd Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Fri, 19 Aug 2022 19:32:54 +0000 Subject: [PATCH 11/18] DAOS-10830 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 012e1d1adde..962b535c71b 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -131,11 +131,10 @@ def test_ls(self): self.add_container(pool=self.pool) # Find the vos file name. e.g., /mnt/daos//vos-0. - vos_file = self.get_vos_file_path() - host = NodeSet(self.hostlist_servers[0]) ddb_command = DdbCommand( - server_host=host, path=self.bin, mount_point="/mnt/daos", - pool_uuid=self.pool.uuid, vos_file=vos_file) + server_host=NodeSet(self.hostlist_servers[0]), path=self.bin, + mount_point="/mnt/daos", pool_uuid=self.pool.uuid, + vos_file=self.get_vos_file_path()) errors = [] @@ -182,9 +181,9 @@ def test_ls(self): actual_object_count = len(match) expected_object_count = self.params.get("object_count", "/run/*") if actual_object_count != expected_object_count: - msg = "Unexpected object count! Expected = {}; Actual = {}".format( - expected_object_count, actual_object_count) - errors.append(msg) + errors.append( + "Unexpected object count! Expected = {}; Actual = {}".format( + expected_object_count, actual_object_count)) # 3. Verify there are two dkeys for every object. Also verify the dkey string and # the size. From 0402978296d0150c5748d4f7ddd5fa807a11240a Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Fri, 19 Aug 2022 19:40:05 +0000 Subject: [PATCH 12/18] DAOS-10830 test: Fix pylint Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index 962b535c71b..fe6680e375d 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -215,10 +215,10 @@ def test_ls(self): # Verify the dkey size field. for dkey in match: dkey_string = dkey[1] - dkey_size = int(dkey[2]) - if len(dkey_string) != dkey_size: + # int(dkey[2]) is the dkey size; avoid pylint error. + if len(dkey_string) != int(dkey[2]): msg = ("Wrong dkey size! obj_index = {}. String = {}; " - "Size = {}").format(obj_index, dkey_string, dkey_size) + "Size = {}").format(obj_index, dkey_string, int(dkey[2])) errors.append(msg) # Verify there are two dkeys for every object. From ccae3e6871489b439d5af5652913a9fd1ff879a7 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Tue, 23 Aug 2022 13:21:00 +0000 Subject: [PATCH 13/18] DAOS-10830 test: Add capability to move saved and loaded files across test node and server node Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.py | 48 ++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/cat_recovery/ddb.py index fe6680e375d..8466f8fa1ac 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/cat_recovery/ddb.py @@ -10,8 +10,11 @@ from apricot import TestWithServers from pydaos.raw import IORequest, DaosObjClass -from general_utils import create_string_buffer, report_errors, run_pcmd +from general_utils import create_string_buffer, report_errors, run_pcmd, \ + distribute_files, DaosTestError, get_clush_command, run_command from ddb_utils import DdbCommand +from exception_utils import CommandFailure + SAMPLE_DKEY = "Sample dkey" SAMPLE_AKEY = "Sample akey" @@ -108,6 +111,34 @@ def get_vos_file_path(self): return vos_file + def copy_remote_to_local(self, remote_file_path): + """Copy the given file from the server node to the local test node and retrieve + the original name. + + Args: + remote_file_path (str): File path to copy to local. + """ + # Use clush --rcopy to copy the file from the remote server node to the local test + # node. clush will append . to the file when copying. + args = "--rcopy {} --dest {}".format(remote_file_path, self.test_dir) + clush_command = get_clush_command(hosts=self.hostlist_servers[0], args=args) + try: + run_command(command=clush_command) + except DaosTestError as error: + raise CommandFailure( + "ERROR: Copying {} from {}: {}".format( + remote_file_path, self.hostlist_servers[0], error)) from error + + # Remove the appended . from the copied file. + current_file_path = "".join([remote_file_path, ".", self.hostlist_servers[0]]) + mv_command = "mv {} {}".format(current_file_path, remote_file_path) + try: + run_command(command=mv_command) + except DaosTestError as error: + raise CommandFailure( + "ERROR: Moving {} to {}: {}".format( + current_file_path, remote_file_path, error)) from error + def test_ls(self): """Test ddb ls. @@ -497,11 +528,22 @@ def test_load(self): pool_uuid=self.pool.uuid, vos_file=vos_file) # 5. Load new data into [0]/[0]/[0]/[0] + # Create a file in test node. load_file_path = os.path.join(self.test_dir, "new_data.txt") new_data = "New akey data 0123456789" with open(load_file_path, "w") as file: file.write(new_data) + # Copy the created file to server node. + try: + distribute_files( + hosts=host, source=load_file_path, destination=load_file_path, + mkdir=False) + except DaosTestError as error: + raise CommandFailure( + "ERROR: Copying new_data.txt to {}: {}".format(host, error)) from error + + # The file with the new data is ready. Run ddb load. ddb_command.load(component_path="[0]/[0]/[0]/[0]", load_file_path=load_file_path) # 6. Restart the server. @@ -576,6 +618,10 @@ def test_dump_value(self): ddb_command.dump_value( component_path="[0]/[0]/[0]/[1]", out_file_path=akey2_file_path) + # Copy them from server node to test node. + self.copy_remote_to_local(remote_file_path=akey1_file_path) + self.copy_remote_to_local(remote_file_path=akey2_file_path) + # 6. Verify the content of the files. actual_akey1_data = None with open(akey1_file_path, "r") as file: From f21fc0b40bedcda2899816d5e5a18d94aca2e9d0 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Wed, 24 Aug 2022 10:37:26 +0000 Subject: [PATCH 14/18] DAOS-10830 test: Set False to start_servers_once Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/cat_recovery/ddb.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tests/ftest/cat_recovery/ddb.yaml b/src/tests/ftest/cat_recovery/ddb.yaml index 92a047e15e8..a009cbc1d69 100644 --- a/src/tests/ftest/cat_recovery/ddb.yaml +++ b/src/tests/ftest/cat_recovery/ddb.yaml @@ -8,6 +8,13 @@ server_config: servers: targets: 1 +# In CI, all tests in ddb.py are ran in a single launch.py execution. In that case, the +# test_dir (/var/tmp/daos_testing/) in the server node will not be created +# for each test if "start_servers_once: False" isn't set. test_load() needs this +# directory, so we need to set it. +setup: + start_servers_once: False + pool: control_method: dmg scm_size: 1G From 5d8abad1a4a4a996929372880e6bbc199ec22730 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Thu, 25 Aug 2022 15:06:59 +0000 Subject: [PATCH 15/18] DAOS-10830 test: Move insert_objects and copy_remote_to_local to general_utils.py Rename file, test methods, and test tags. Generate random base string for dkey, akey, and data. Fix pylint. Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/SConscript | 4 +- .../ftest/{cat_recovery => recovery}/ddb.py | 268 +++++++----------- .../ftest/{cat_recovery => recovery}/ddb.yaml | 4 - src/tests/ftest/util/daos_utils.py | 4 +- src/tests/ftest/util/general_utils.py | 98 ++++++- utils/cq/words.dict | 1 + 6 files changed, 204 insertions(+), 175 deletions(-) rename src/tests/ftest/{cat_recovery => recovery}/ddb.py (73%) rename src/tests/ftest/{cat_recovery => recovery}/ddb.yaml (91%) diff --git a/src/tests/ftest/SConscript b/src/tests/ftest/SConscript index 9f94dbbcf83..9dff4c1348b 100644 --- a/src/tests/ftest/SConscript +++ b/src/tests/ftest/SConscript @@ -10,12 +10,12 @@ def scons(): env.Install(ftest_install_dir, Glob('*.*')) - dirs = ['aggregation', 'fault_injection', 'cat_recovery', 'checksum', + dirs = ['aggregation', 'fault_injection', 'checksum', 'container', 'control', 'dfuse', 'dtx', 'daos_perf', 'daos_racer', 'daos_vol', 'daos_test', 'data', 'fault_domain', 'io', 'ior', 'mdtest', 'network', 'nvme', 'mpiio', - 'object', 'osa', 'pool', 'rebuild', 'security', + 'object', 'osa', 'pool', 'rebuild', 'recovery', 'security', 'server', 'soak', 'erasurecode', 'datamover', 'scripts', 'dbench', 'harness', 'telemetry', 'deployment', 'performance', 'scrubber'] diff --git a/src/tests/ftest/cat_recovery/ddb.py b/src/tests/ftest/recovery/ddb.py similarity index 73% rename from src/tests/ftest/cat_recovery/ddb.py rename to src/tests/ftest/recovery/ddb.py index 8466f8fa1ac..1b18c49795c 100644 --- a/src/tests/ftest/cat_recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -4,23 +4,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -import ctypes import re from ClusterShell.NodeSet import NodeSet from apricot import TestWithServers -from pydaos.raw import IORequest, DaosObjClass -from general_utils import create_string_buffer, report_errors, run_pcmd, \ - distribute_files, DaosTestError, get_clush_command, run_command +from general_utils import report_errors, run_pcmd, insert_objects, \ + distribute_files, DaosTestError, get_random_string, copy_remote_to_local from ddb_utils import DdbCommand from exception_utils import CommandFailure -SAMPLE_DKEY = "Sample dkey" -SAMPLE_AKEY = "Sample akey" -SAMPLE_DATA = "Sample data" - - class DdbTest(TestWithServers): """Test ddb subcommands. @@ -30,63 +23,16 @@ class DdbTest(TestWithServers): def __init__(self, *args, **kwargs): """Initialize a DdbTest object.""" super().__init__(*args, **kwargs) - self.ioreqs = [] - self.dkeys = [] - self.akeys = [] - self.data_list = [] - - def insert_objects(self, object_count, dkey_count, akey_count): - """Insert objects, dkeys, akeys, and data into the container. - - Inserted objects: self.ioreqs - Inserted dkeys: self.dkeys - Inserted akeys: self.akeys - Inserted data: self.data_list - - Args: - object_count (int): Number of objects to insert. - dkey_count (int): Number of dkeys to insert. - akey_count (int): Number of akeys to insert. - """ - self.container.open() - - for obj_index in range(object_count): - # Insert object. - self.ioreqs.append(IORequest( - context=self.context, container=self.container.container, obj=None, - objtype=DaosObjClass.OC_S1)) - - for dkey_index in range(dkey_count): - # Prepare the dkey to insert into the object. - dkey_str = " ".join( - [SAMPLE_DKEY, str(obj_index), str(dkey_index)]).encode("utf-8") - self.dkeys.append( - create_string_buffer(value=dkey_str, size=len(dkey_str))) - - for akey_index in range(akey_count): - # Prepare the akey to insert into the dkey. - akey_str = " ".join( - [SAMPLE_AKEY, str(obj_index), str(dkey_index), - str(akey_index)]).encode("utf-8") - self.akeys.append( - create_string_buffer(value=akey_str, size=len(akey_str))) - - # Prepare the data to insert into the akey. - data_str = " ".join( - [SAMPLE_DATA, str(obj_index), str(dkey_index), - str(akey_index)]).encode("utf-8") - self.data_list.append( - create_string_buffer(value=data_str, size=len(data_str))) - c_size = ctypes.c_size_t(ctypes.sizeof(self.data_list[-1])) - - # Insert dkeys, akeys, and the data. - self.ioreqs[-1].single_insert( - dkey=self.dkeys[-1], akey=self.akeys[-1], - value=self.data_list[-1], size=c_size) + # Generate random keys and data to insert into the object. + self.random_dkey = get_random_string(10) + self.random_akey = get_random_string(10) + self.random_data = get_random_string(10) def get_vos_file_path(self): """Get the VOS file path. + If there are multiple VOS files, returns the first file obtained by "ls". + Returns: str: VOS file path such as /mnt/daos//vos-0 @@ -96,50 +42,18 @@ def get_vos_file_path(self): command = " ".join(["sudo", "ls", vos_path]) cmd_out = run_pcmd(hosts=hosts, command=command) - vos_file = None + # return vos_file for file in cmd_out[0]["stdout"]: # Assume the VOS file has "vos" in the file name. if "vos" in file: - vos_file = file - break - - if not vos_file: - self.fail( - "vos file wasn't found in /mnt/daos/{}".format(self.pool.uuid.lower())) - else: - self.log.info("vos_file: %s", vos_file) - - return vos_file - - def copy_remote_to_local(self, remote_file_path): - """Copy the given file from the server node to the local test node and retrieve - the original name. + self.log.info("vos_file: %s", file) + return file - Args: - remote_file_path (str): File path to copy to local. - """ - # Use clush --rcopy to copy the file from the remote server node to the local test - # node. clush will append . to the file when copying. - args = "--rcopy {} --dest {}".format(remote_file_path, self.test_dir) - clush_command = get_clush_command(hosts=self.hostlist_servers[0], args=args) - try: - run_command(command=clush_command) - except DaosTestError as error: - raise CommandFailure( - "ERROR: Copying {} from {}: {}".format( - remote_file_path, self.hostlist_servers[0], error)) from error + self.fail("vos file wasn't found in /mnt/daos/{}".format(self.pool.uuid.lower())) - # Remove the appended . from the copied file. - current_file_path = "".join([remote_file_path, ".", self.hostlist_servers[0]]) - mv_command = "mv {} {}".format(current_file_path, remote_file_path) - try: - run_command(command=mv_command) - except DaosTestError as error: - raise CommandFailure( - "ERROR: Moving {} to {}: {}".format( - current_file_path, remote_file_path, error)) from error + return None # to appease pylint - def test_ls(self): + def test_recovery_ddb_ls(self): """Test ddb ls. 1. Verify container UUID. @@ -154,8 +68,8 @@ def test_ls(self): :avocado: tags=all,weekly_regression :avocado: tags=vm - :avocado: tags=cat_rec - :avocado: tags=ddb,ddb_ls + :avocado: tags=recovery + :avocado: tags=ddb,test_recovery_ddb_ls """ # Create a pool and a container. self.add_pool() @@ -170,11 +84,13 @@ def test_ls(self): errors = [] # Insert objects with API. - object_count = self.params.get("object_count", "/run/*") - dkey_count = self.params.get("dkey_count", "/run/*") - akey_count = self.params.get("akey_count", "/run/*") - self.insert_objects( - object_count=object_count, dkey_count=dkey_count, akey_count=akey_count) + object_count = 5 + dkey_count = 2 + akey_count = 1 + insert_objects( + context=self.context, container=self.container, object_count=5, + dkey_count=2, akey_count=1, base_dkey=self.random_dkey, + base_akey=self.random_akey, base_data=self.random_data) # Need to stop the server to use ddb. self.get_dmg_command().system_stop() @@ -186,10 +102,10 @@ def test_ls(self): # stdout is a list which contains each line as separate element. Concatenate them # to single string so that we can apply regex. ls_out = "\n".join(cmd_result[0]["stdout"]) - match = re.findall(r"(\[\d+\])\s+([a-f0-9\-]+)", ls_out) - self.log.info("List container match = %s", match) + match = re.search(r"\[\d+\]\s+([a-f0-9\-]+)", ls_out) + self.log.info("Container UUID from ddb ls = %s", match.group(1)) - actual_uuid = match[0][1].lower() + actual_uuid = match.group(1).lower() expected_uuid = self.container.uuid.lower() if actual_uuid != expected_uuid: msg = "Unexpected container UUID! Expected = {}; Actual = {}".format( @@ -236,12 +152,12 @@ def test_ls(self): actual_dkey_1 = " ".join(match[0][1].split()) actual_dkey_2 = " ".join(match[1][1].split()) # We're not testing the numbers in the string because it's not deterministic. - if SAMPLE_DKEY not in actual_dkey_1: + if self.random_dkey not in actual_dkey_1: msg = ("Unexpected dkey! obj_i = {}. Expected = {}; " - "Actual = {}").format(obj_index, SAMPLE_DKEY, actual_dkey_1) - if SAMPLE_DKEY not in actual_dkey_2: + "Actual = {}").format(obj_index, self.random_dkey, actual_dkey_1) + if self.random_dkey not in actual_dkey_2: msg = ("Unexpected dkey! obj_i = {}. Expected = {}; " - "Actual = {}").format(obj_index, SAMPLE_DKEY, actual_dkey_2) + "Actual = {}").format(obj_index, self.random_dkey, actual_dkey_2) # Verify the dkey size field. for dkey in match: @@ -280,10 +196,10 @@ def test_ls(self): # Verify akey string. As in dkey, ignore the numbers at the end. actual_akey = " ".join(match[0][1].split()) - if SAMPLE_AKEY not in actual_akey: + if self.random_akey not in actual_akey: msg = ("Unexpected akey! obj_index = {}; dkey_index = {}; " "Expected = {}; Actual = {}").format( - obj_index, dkey_index, SAMPLE_AKEY, actual_akey) + obj_index, dkey_index, self.random_akey, actual_akey) errors.append(msg) # Verify akey size. @@ -314,7 +230,7 @@ def test_ls(self): # [0] Single Value (Length: 17 bytes) match = re.findall(r"Length:\s+(\d+)\s+bytes", ls_out) data_length = int(match[0]) - expected_data_length = len(SAMPLE_DATA) + 6 + expected_data_length = len(self.random_data) + 6 if data_length != expected_data_length: msg = "Wrong data length! {}; Expected = {}; Actual = {}".format( path_stat, expected_data_length, data_length) @@ -334,7 +250,7 @@ def test_ls(self): report_errors(test=self, errors=errors) self.log.info("##################") - def test_rm(self): + def test_recovery_ddb_rm(self): """Test rm. 1. Create a pool and a container. Insert objects, dkeys, and akeys. @@ -358,26 +274,32 @@ def test_rm(self): :avocado: tags=all,weekly_regression :avocado: tags=vm - :avocado: tags=cat_rec - :avocado: tags=ddb,ddb_rm + :avocado: tags=recovery + :avocado: tags=ddb,test_recovery_ddb_rm """ # 1. Create a pool and a container. Insert objects, dkeys, and akeys. self.add_pool(connect=True) self.add_container(pool=self.pool) - # Insert one object with one dkey with API. - self.insert_objects(object_count=1, dkey_count=1, akey_count=2) + # Insert one object with one dkey and one akey with API. + obj_dataset = insert_objects( + context=self.context, container=self.container, object_count=1, + dkey_count=1, akey_count=2, base_dkey=self.random_dkey, + base_akey=self.random_akey, base_data=self.random_data) + ioreqs = obj_dataset[0] + dkeys_inserted = obj_dataset[1] + akeys_inserted = obj_dataset[2] # For debugging/reference, check that the dkey and the akey we just inserted are # returned from the API. - akeys = self.ioreqs[0].list_akey(dkey=self.dkeys[0]) - self.log.info("akeys from API (before) = %s", akeys) - dkeys = self.ioreqs[0].list_dkey() - self.log.info("dkeys from API (before) = %s", dkeys) + akeys_api = ioreqs[0].list_akey(dkey=dkeys_inserted[0]) + self.log.info("akeys from API (before) = %s", akeys_api) + dkeys_api = ioreqs[0].list_dkey() + self.log.info("dkeys from API (before) = %s", dkeys_api) # For debugging/reference, check that the object was inserted using daos command. list_obj_out = self.get_daos_command().container_list_objects( - pool=self.pool.uuid, cont=self.container.uuid) + pool=self.pool.identifier, cont=self.container.uuid) self.log.info("Object list (before) = %s", list_obj_out["response"]) # 2. Need to stop the server to use ddb. @@ -399,20 +321,20 @@ def test_rm(self): dmg_command.system_start() # 6. Reset the object, container, and pool to use the API after server restart. - self.ioreqs[0].obj.close() + ioreqs[0].obj.close() self.container.close() self.pool.disconnect() self.pool.connect() self.container.open() - self.ioreqs[0].obj.open() + ioreqs[0].obj.open() # 7. Call list_akey() in pydaos API to verify that the akey was removed. - akeys = self.ioreqs[0].list_akey(dkey=self.dkeys[0]) - self.log.info("akeys from API (after) = %s", akeys) + akeys_api = ioreqs[0].list_akey(dkey=dkeys_inserted[0]) + self.log.info("akeys from API (after) = %s", akeys_api) errors = [] - expected_len = len(self.akeys) - 1 - actual_len = len(akeys) + expected_len = len(akeys_inserted) - 1 + actual_len = len(akeys_api) if actual_len != expected_len: msg = ("Unexpected number of akeys after ddb rm! " "Expected = {}; Actual = {}").format(expected_len, actual_len) @@ -429,19 +351,19 @@ def test_rm(self): dmg_command.system_start() # 11. Reset the object, container, and pool to use the API after server restart. - self.ioreqs[0].obj.close() + ioreqs[0].obj.close() self.container.close() self.pool.disconnect() self.pool.connect() self.container.open() - self.ioreqs[0].obj.open() + ioreqs[0].obj.open() # 12. Call list_dkey() in pydaos API to verify that the dkey was removed. - dkeys = self.ioreqs[0].list_dkey() - self.log.info("dkeys from API (after) = %s", dkeys) + dkeys_api = ioreqs[0].list_dkey() + self.log.info("dkeys from API (after) = %s", dkeys_api) - expected_len = len(self.dkeys) - 1 - actual_len = len(dkeys) + expected_len = len(dkeys_inserted) - 1 + actual_len = len(dkeys_api) if actual_len != expected_len: msg = ("Unexpected number of dkeys after ddb rm! " "Expected = {}; Actual = {}").format(expected_len, actual_len) @@ -466,11 +388,11 @@ def test_rm(self): # 17. Call "daos container list-objects " to verify that # the object was removed. list_obj_out = self.get_daos_command().container_list_objects( - pool=self.pool.uuid, cont=self.container.uuid) + pool=self.pool.identifier, cont=self.container.uuid) obj_list = list_obj_out["response"] self.log.info("Object list (after) = %s", obj_list) - expected_len = len(self.ioreqs) - 1 + expected_len = len(ioreqs) - 1 if obj_list: actual_len = len(obj_list) else: @@ -484,7 +406,7 @@ def test_rm(self): report_errors(test=self, errors=errors) self.log.info("##################") - def test_load(self): + def test_recovery_ddb_load(self): """Test ddb load. 1. Create a pool and a container. @@ -498,22 +420,28 @@ def test_load(self): :avocado: tags=all,weekly_regression :avocado: tags=vm - :avocado: tags=cat_rec - :avocado: tags=ddb,ddb_load + :avocado: tags=recovery + :avocado: tags=ddb,test_recovery_ddb_load """ # 1. Create a pool and a container. self.add_pool(connect=True) self.add_container(pool=self.pool) # 2. Insert one object with one dkey with API. - self.insert_objects(object_count=1, dkey_count=1, akey_count=1) + obj_dataset = insert_objects( + context=self.context, container=self.container, object_count=1, + dkey_count=1, akey_count=1, base_dkey=self.random_dkey, + base_akey=self.random_akey, base_data=self.random_data) + ioreqs = obj_dataset[0] + dkeys_inserted = obj_dataset[1] + akeys_inserted = obj_dataset[2] + data_list = obj_dataset[3] # For debugging/reference, call single_fetch and get the data just inserted. - data_size = len(self.data_list[0]) - # Pass in data_size + 1 to single_fetch to avoid the no-space error. - data_size += 1 - data = self.ioreqs[0].single_fetch( - dkey=self.dkeys[0], akey=self.akeys[0], size=data_size) + # Pass in size + 1 to single_fetch to avoid the no-space error. + data_size = len(data_list[0]) + 1 + data = ioreqs[0].single_fetch( + dkey=dkeys_inserted[0], akey=akeys_inserted[0], size=data_size) self.log.info("data (before) = %s", data.value.decode('utf-8')) # 3. Stop the server to use ddb. @@ -550,18 +478,17 @@ def test_load(self): dmg_command.system_start() # 7. Reset the object, container, and pool to use the API after server restart. - self.ioreqs[0].obj.close() + ioreqs[0].obj.close() self.container.close() self.pool.disconnect() self.pool.connect() self.container.open() - self.ioreqs[0].obj.open() + ioreqs[0].obj.open() # 8. Verify the data in the akey with single_fetch(). - data_size = len(new_data) - data_size += 1 - data = self.ioreqs[0].single_fetch( - dkey=self.dkeys[0], akey=self.akeys[0], size=data_size) + data_size = len(new_data) + 1 + data = ioreqs[0].single_fetch( + dkey=dkeys_inserted[0], akey=akeys_inserted[0], size=data_size) actual_data = data.value.decode('utf-8') self.log.info("data (after) = %s", actual_data) @@ -575,7 +502,7 @@ def test_load(self): report_errors(test=self, errors=errors) self.log.info("##################") - def test_dump_value(self): + def test_recovery_ddb_dump_value(self): """Test ddb dump_value. 1. Create a pool and a container. @@ -589,15 +516,20 @@ def test_dump_value(self): :avocado: tags=all,weekly_regression :avocado: tags=vm - :avocado: tags=cat_rec - :avocado: tags=ddb,ddb_dump_value + :avocado: tags=recovery + :avocado: tags=ddb,test_recovery_ddb_dump_value """ # 1. Create a pool and a container. self.add_pool(connect=True) self.add_container(pool=self.pool) # 2. Insert one object with one dkey with API. - self.insert_objects(object_count=1, dkey_count=1, akey_count=2) + obj_dataset = insert_objects( + context=self.context, container=self.container, object_count=1, + dkey_count=1, akey_count=2, base_dkey=self.random_dkey, + base_akey=self.random_akey, base_data=self.random_data) + ioreqs = obj_dataset[0] + data_list = obj_dataset[3] # 3. Stop the server to use ddb. dmg_command = self.get_dmg_command() @@ -618,9 +550,13 @@ def test_dump_value(self): ddb_command.dump_value( component_path="[0]/[0]/[0]/[1]", out_file_path=akey2_file_path) - # Copy them from server node to test node. - self.copy_remote_to_local(remote_file_path=akey1_file_path) - self.copy_remote_to_local(remote_file_path=akey2_file_path) + # Copy them from remote server node to local test node. + copy_remote_to_local( + remote_file_path=akey1_file_path, test_dir=self.test_dir, + remote=self.hostlist_servers[0]) + copy_remote_to_local( + remote_file_path=akey2_file_path, test_dir=self.test_dir, + remote=self.hostlist_servers[0]) # 6. Verify the content of the files. actual_akey1_data = None @@ -633,7 +569,7 @@ def test_dump_value(self): errors = [] str_data_list = [] # Convert the data to string. - for data in self.data_list: + for data in data_list: str_data_list.append(data.value.decode("utf-8")) # Verify that we were able to obtain the data and akey1 and akey2 aren't the same. if actual_akey1_data is None or actual_akey2_data is None or \ @@ -654,12 +590,12 @@ def test_dump_value(self): dmg_command.system_start() # 8. Reset the object, container, and pool to prepare for the cleanup. - self.ioreqs[0].obj.close() + ioreqs[0].obj.close() self.container.close() self.pool.disconnect() self.pool.connect() self.container.open() - self.ioreqs[0].obj.open() + ioreqs[0].obj.open() self.log.info("##### Errors #####") report_errors(test=self, errors=errors) diff --git a/src/tests/ftest/cat_recovery/ddb.yaml b/src/tests/ftest/recovery/ddb.yaml similarity index 91% rename from src/tests/ftest/cat_recovery/ddb.yaml rename to src/tests/ftest/recovery/ddb.yaml index a009cbc1d69..13b8e702985 100644 --- a/src/tests/ftest/cat_recovery/ddb.yaml +++ b/src/tests/ftest/recovery/ddb.yaml @@ -21,7 +21,3 @@ pool: container: control_method: API - -object_count: 5 -dkey_count: 2 -akey_count: 1 diff --git a/src/tests/ftest/util/daos_utils.py b/src/tests/ftest/util/daos_utils.py index 6f498f4b049..e4cf5b2e6e3 100644 --- a/src/tests/ftest/util/daos_utils.py +++ b/src/tests/ftest/util/daos_utils.py @@ -608,8 +608,8 @@ def container_list_objects(self, pool, cont, sys_name=None): """Call daos container list-objects. Args: - pool (str): Pool UUID. - cont (str): Container UUID. + pool (str): Pool UUID or label + cont (str): Container UUID or label sys_name (str, optional): DAOS system name context for servers. Defaults to None. diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 83a6e0273f0..24440e566a8 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/python """ (C) Copyright 2018-2022 Intel Corporation. @@ -24,6 +23,7 @@ from avocado.utils import process from ClusterShell.Task import task_self from ClusterShell.NodeSet import NodeSet, NodeSetParseError +from pydaos.raw import IORequest, DaosObjClass class DaosTestError(Exception): @@ -1525,3 +1525,99 @@ def set_avocado_config_value(section, key, value): settings.update_option(".".join([section, key]), value) else: settings.config.set(section, key, str(value)) + + +def insert_objects(context, container, object_count, dkey_count, akey_count, base_dkey, + base_akey, base_data): + """Insert objects, dkeys, akeys, and data into the container. + + Inserted objects: self.ioreqs + Inserted dkeys: self.dkeys + Inserted akeys: self.akeys + Inserted data: self.data_list + + Args: + context (DaosContext): + container (TestContainer): Container to insert objects. + object_count (int): Number of objects to insert. + dkey_count (int): Number of dkeys to insert. + akey_count (int): Number of akeys to insert. + base_dkey (str): Base dkey. Index numbers will be appended to it. + base_akey (str):Base akey. Index numbers will be appended to it. + base_data (str):Base data that goes inside akey. Index numbers will be appended + to it. + + Returns: + tuple: Inserted objects, dkeys, akeys, and data as (ioreqs, dkeys, akeys, + data_list) + + """ + ioreqs = [] + dkeys = [] + akeys = [] + data_list = [] + + container.open() + + for obj_index in range(object_count): + # Insert object. + ioreqs.append(IORequest( + context=context, container=container.container, obj=None, + objtype=DaosObjClass.OC_S1)) + + for dkey_index in range(dkey_count): + # Prepare the dkey to insert into the object. + dkey_str = " ".join( + [base_dkey, str(obj_index), str(dkey_index)]).encode("utf-8") + dkeys.append(create_string_buffer(value=dkey_str, size=len(dkey_str))) + + for akey_index in range(akey_count): + # Prepare the akey to insert into the dkey. + akey_str = " ".join( + [base_akey, str(obj_index), str(dkey_index), + str(akey_index)]).encode("utf-8") + akeys.append(create_string_buffer(value=akey_str, size=len(akey_str))) + + # Prepare the data to insert into the akey. + data_str = " ".join( + [base_data, str(obj_index), str(dkey_index), + str(akey_index)]).encode("utf-8") + data_list.append(create_string_buffer(value=data_str, size=len(data_str))) + c_size = ctypes.c_size_t(ctypes.sizeof(data_list[-1])) + + # Insert dkeys, akeys, and the data. + ioreqs[-1].single_insert( + dkey=dkeys[-1], akey=akeys[-1], value=data_list[-1], size=c_size) + + return (ioreqs, dkeys, akeys, data_list) + + +def copy_remote_to_local(remote_file_path, test_dir, remote): + """Copy the given file from the server node to the local test node and retrieve + the original name. + + Args: + remote_file_path (str): File path to copy to local. + test_dir (str): Test directory. Usually self.test_dir. + remote (str): Remote hostname to copy file from. + """ + # Use clush --rcopy to copy the file from the remote server node to the local test + # node. clush will append . to the file when copying. + args = "--rcopy {} --dest {}".format(remote_file_path, test_dir) + clush_command = get_clush_command(hosts=remote, args=args) + try: + run_command(command=clush_command) + except DaosTestError as error: + print("ERROR: Copying {} from {}: {}".format(remote_file_path, remote, error)) + raise error + + # Remove the appended . from the copied file. + current_file_path = "".join([remote_file_path, ".", remote]) + mv_command = "mv {} {}".format(current_file_path, remote_file_path) + try: + run_command(command=mv_command) + except DaosTestError as error: + print( + "ERROR: Moving {} to {}: {}".format( + current_file_path, remote_file_path, error)) + raise error diff --git a/utils/cq/words.dict b/utils/cq/words.dict index 5435b2b3de7..8c7f1916622 100644 --- a/utils/cq/words.dict +++ b/utils/cq/words.dict @@ -205,6 +205,7 @@ iod iodepth ioengine ior +ioreqs iotype iov ip From 327460546f368620f824e5666795c4fa17cdd810 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Thu, 25 Aug 2022 15:16:55 +0000 Subject: [PATCH 16/18] DAOS-10830 test: Update comment Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/util/general_utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 24440e566a8..ff71298f3bd 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -1531,11 +1531,6 @@ def insert_objects(context, container, object_count, dkey_count, akey_count, bas base_akey, base_data): """Insert objects, dkeys, akeys, and data into the container. - Inserted objects: self.ioreqs - Inserted dkeys: self.dkeys - Inserted akeys: self.akeys - Inserted data: self.data_list - Args: context (DaosContext): container (TestContainer): Container to insert objects. From 4a3bfddd7fa9baa9d55f0eb33fbe7ef1b4133f8a Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Thu, 25 Aug 2022 16:39:04 +0000 Subject: [PATCH 17/18] DAOS-10830 test: Use correct variable for expected object count Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index 1b18c49795c..e735cfe6956 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -126,11 +126,10 @@ def test_recovery_ddb_ls(self): self.log.info("List objects match = %s", match) actual_object_count = len(match) - expected_object_count = self.params.get("object_count", "/run/*") - if actual_object_count != expected_object_count: + if actual_object_count != object_count: errors.append( "Unexpected object count! Expected = {}; Actual = {}".format( - expected_object_count, actual_object_count)) + object_count, actual_object_count)) # 3. Verify there are two dkeys for every object. Also verify the dkey string and # the size. From 1a5602008731ea645eaae3acc9fc5a5f3ba22578 Mon Sep 17 00:00:00 2001 From: Makito Kano Date: Fri, 26 Aug 2022 16:43:41 +0000 Subject: [PATCH 18/18] DAOS-10830 test: Fix variable usage Skip-unit-tests: true Skip-fault-injection-test: true Test-tag: test_pool_info_query ddb Required-githooks: true Signed-off-by: Makito Kano --- src/tests/ftest/recovery/ddb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index e735cfe6956..2cb0dbfe486 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -88,8 +88,8 @@ def test_recovery_ddb_ls(self): dkey_count = 2 akey_count = 1 insert_objects( - context=self.context, container=self.container, object_count=5, - dkey_count=2, akey_count=1, base_dkey=self.random_dkey, + context=self.context, container=self.container, object_count=object_count, + dkey_count=dkey_count, akey_count=akey_count, base_dkey=self.random_dkey, base_akey=self.random_akey, base_data=self.random_data) # Need to stop the server to use ddb.