From 98bef4270f8ec23631f8572baf51bc370eb208e9 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Fri, 2 Dec 2022 14:38:00 +0000 Subject: [PATCH 01/23] Initial commit of 3 files from Vivien's branch These 3 files came from commit ID e391dadb5d759ce3044e175260f028eff8efec9f --- .../tests/yamltests/ClustersDefinitions.py | 254 ++++++ scripts/tests/yamltests/YamlFixes.py | 84 ++ scripts/tests/yamltests/YamlParser.py | 788 ++++++++++++++++++ 3 files changed, 1126 insertions(+) create mode 100644 scripts/tests/yamltests/ClustersDefinitions.py create mode 100644 scripts/tests/yamltests/YamlFixes.py create mode 100644 scripts/tests/yamltests/YamlParser.py diff --git a/scripts/tests/yamltests/ClustersDefinitions.py b/scripts/tests/yamltests/ClustersDefinitions.py new file mode 100644 index 00000000000000..f08eabbeee581f --- /dev/null +++ b/scripts/tests/yamltests/ClustersDefinitions.py @@ -0,0 +1,254 @@ +# +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import xml.etree.ElementTree as ET +import glob +import time + + +class ClustersDefinitions: + def __init__(self, clusters_dir): + self.__clusterIdToName = {} + self.__commandIdToName = {} + self.__attributeIdToName = {} + self.__globalAttributeIdToName = {} + self.__eventIdToName = {} + + self.__commands = {} + self.__responses = {} + self.__attributes = {} + self.__events = {} + self.__structs = {} + self.__enums = {} + self.__bitmaps = {} + + clusters_files = glob.glob(clusters_dir + '**/*.xml', recursive=True) + for cluster_file in clusters_files: + tree = ET.parse(cluster_file) + + root = tree.getroot() + + structs = root.findall('struct') + for struct in structs: + self.__structs[struct.get('name').lower()] = struct + + enums = root.findall('enum') + for enum in enums: + self.__enums[enum.get('name').lower()] = enum + + bitmaps = root.findall('bitmap') + for bitmap in bitmaps: + self.__bitmaps[bitmap.get('name').lower()] = bitmap + + global_elements = root.find('global') + if global_elements: + global_attributes = global_elements.findall('attribute') + + for attribute in global_attributes: + attribute_side = attribute.get('side') + attribute_code = attribute.get('code') + + description = attribute.find('description') + attribute_name = description.text if description is not None else attribute.text + + if attribute_code.startswith('0x') or attribute_code.startswith('0X'): + attribute_code = int(attribute_code, base=16) + self.__globalAttributeIdToName[attribute_code] = attribute_name + + if attribute_side == 'server': + self.__attributes[attribute_name.lower()] = attribute + + cluster = root.find('cluster') + if not cluster: + continue + + cluster_code = cluster.find('code').text + cluster_name = cluster.find('name').text + + if cluster_code.startswith('0x') or cluster_code.startswith('0X'): + cluster_code = int(cluster_code, base=16) + + self.__clusterIdToName[cluster_code] = cluster_name + self.__commandIdToName[cluster_code] = {} + self.__attributeIdToName[cluster_code] = {} + self.__eventIdToName[cluster_code] = {} + + commands = cluster.findall('command') + for command in commands: + command_source = command.get('source') + command_code = command.get('code') + command_name = command.get('name') + + base = 16 if command_code.startswith('0x') or command_code.startswith('0X') else 10 + command_code = int(command_code, base=base) + self.__commandIdToName[cluster_code][command_code] = command_name + + if command_source == 'client': + self.__commands[command_name.lower()] = command + elif command_source == 'server': + # The name is not converted to lowercase here + self.__responses[command_name] = command + + attributes = cluster.findall('attribute') + for attribute in attributes: + attribute_side = attribute.get('side') + attribute_code = attribute.get('code') + + description = attribute.find('description') + attribute_name = description.text if description is not None else attribute.text + + base = 16 if attribute_code.startswith('0x') or attribute_code.startswith('0X') else 10 + attribute_code = int(attribute_code, base=base) + self.__attributeIdToName[cluster_code][attribute_code] = attribute_name + + if attribute_side == 'server': + self.__attributes[attribute_name.lower()] = attribute + + events = cluster.findall('event') + for event in events: + event_side = event.get('side') + event_code = event.get('code') + + description = event.find('description') + event_name = description.text if description is not None else event.text + + base = 16 if event_code.startswith('0x') or event_code.startswith('0X') else 10 + event_code = int(event_code, base=base) + self.__eventIdToName[cluster_code][event_code] = event_name + + if event_side == 'server': + self.__events[event_name.lower()] = event + + def get_cluster_name(self, cluster_id): + return self.__clusterIdToName[cluster_id] + + def get_command_name(self, cluster_id, command_id): + return self.__commandIdToName[cluster_id][command_id] + + def get_attribute_name(self, cluster_id, attribute_id): + if attribute_id in self.__globalAttributeIdToName: + return self.__globalAttributeIdToName[attribute_id] + return self.__attributeIdToName[cluster_id][attribute_id] + + def get_event_name(self, cluster_id, event_id): + return self.__eventIdToName[cluster_id][event_id] + + def get_response_mapping(self, command_name): + if not command_name in self.__responses: + return None + response = self.__responses[command_name] + + args = response.findall('arg') + + mapping = {} + for mapping_index, arg in enumerate(args): + mapping[str(mapping_index)] = {'name': arg.get('name'), 'type': arg.get('type').lower()} + return mapping + + def get_attribute_mapping(self, attribute_name): + if not attribute_name.lower() in self.__attributes: + return None + attribute = self.__attributes[attribute_name.lower()] + + attribute_type = attribute.get('type') + if attribute_type.lower() == 'array': + attribute_type = attribute.get('entryType') + + if not self.get_type_mapping(attribute_type): + return None + + return {'name': attribute_name.lower(), 'type': attribute_type} + + def get_event_mapping(self, event_name): + return None + + def get_type_mapping(self, type_name): + struct = self.__structs.get(type_name.lower()) + if struct is None: + return None + + mapping = {} + + # FabricIndex is a special case where the framework requires it to be passed even if it is not part of the + # requested arguments per spec and not part of the XML definition. + if struct.get('isFabricScoped'): + mapping[str(254)] = {'name': 'FabricIndex', 'type': 'int64u'} + + mapping_index = 0 + + items = struct.findall('item') + for item in items: + if item.get('fieldId'): + mapping_index = int(item.get('fieldId')) + mapping[str(mapping_index)] = {'name': item.get('name'), 'type': item.get('type').lower()} + mapping_index += 1 + + return mapping + + def get_attribute_definition(self, attribute_name): + attribute = self.__attributes.get(attribute_name.lower()) + if attribute is None: + return None + + attribute_type = attribute.get('type').lower() + if attribute_type == 'array': + attribute_type = attribute.get('entryType').lower() + return self.__to_base_type(attribute_type) + + def get_command_args_definition(self, command_name): + command = self.__commands.get(command_name.lower()) + if command is None: + return None + + return self.__args_to_dict(command.findall('arg')) + + def get_response_args_definition(self, command_name): + command = self.__commands.get(command_name.lower()) + if command is None: + return None + + response = self.__responses.get(command.get('response')) + if response is None: + return None + + return self.__args_to_dict(response.findall('arg')) + + def __args_to_dict(self, args): + rv = {} + for item in args: + arg_name = item.get('name') + arg_type = item.get('type').lower() + rv[arg_name] = self.__to_base_type(arg_type) + return rv + + def __to_base_type(self, item_type): + if item_type in self.__bitmaps: + bitmap = self.__bitmaps[item_type] + item_type = bitmap.get('type').lower() + elif item_type in self.__enums: + enum = self.__enums[item_type] + item_type = enum.get('type').lower() + elif item_type in self.__structs: + struct = self.__structs[item_type] + item_type = self.__struct_to_dict(struct) + return item_type + + def __struct_to_dict(self, struct): + type_entry = {} + for item in struct.findall('item'): + item_name = item.get('name') + item_type = item.get('type').lower() + type_entry[item_name] = self.__to_base_type(item_type) + return type_entry diff --git a/scripts/tests/yamltests/YamlFixes.py b/scripts/tests/yamltests/YamlFixes.py new file mode 100644 index 00000000000000..1178090815574b --- /dev/null +++ b/scripts/tests/yamltests/YamlFixes.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +# Some of the YAML files contains some values that has been crafted to avoid some limitations +# of the original JavaScript parser and the C++ generated bits. +# +# This class search the YAML for those changes and convert them back to something agnostic. + + +def try_apply_yaml_cpp_longlong_limitation_fix(value): + # TestCluster contains a nasty hack for C++ in order to avoid a warning: -9223372036854775808 is not a valid way to write a long long in C++ it uses + # "-9223372036854775807LL - 1". This fix replace this hack by -9223372036854775808. + if value == "-9223372036854775807LL - 1": + value = -9223372036854775808 + return value + + +def try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value): + # JavaScript can not represent integers bigger than 9007199254740991. But some of the test may uses values that are bigger + # than this. The current way to workaround this limitation has been to write those numbers as strings by encapsulating them in "". + if type(value) is str: + value = int(value) + return value + + +def try_apply_yaml_float_written_as_strings(value): + if type(value) is str: + value = float(value) + return value + +# The PyYAML implementation does not match float according to the JSON and fails on valid numbers. + + +def try_add_yaml_support_for_scientific_notation_without_dot(loader): + regular_expression = re.compile(u'''^(?: + [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)? + |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+) + |\\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]* + |[-+]?\\.(?:inf|Inf|INF) + |\\.(?:nan|NaN|NAN))$''', re.X) + + loader.add_implicit_resolver( + u'tag:yaml.org,2002:float', + regular_expression, + list(u'-+0123456789.')) + return loader + +# This is a gross hack. The previous runner has a some internal states where an identity match one +# accessory. But this state may not exist in the runner (as in it prevent to have multiple node ids +# associated to a fabric...) so the 'nodeId' needs to be added back manually. + + +def try_update_yaml_node_id_test_runner_state(tests, config): + identities = {'alpha': None if 'nodeId' not in config else config['nodeId']} + + for test in tests: + if not test.isEnabled: + continue + + identity = test.identity + + if test.cluster == 'CommissionerCommands': + if test.command == 'PairWithCode': + for item in test.arguments['values']: + if item['name'] == 'nodeId': + identities[identity] = item['value'] + elif identity is not None and identity in identities: + nodeId = identities[identity] + test.nodeId = nodeId diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py new file mode 100644 index 00000000000000..e4a8c8a72a407b --- /dev/null +++ b/scripts/tests/yamltests/YamlParser.py @@ -0,0 +1,788 @@ +# +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import string + +from . import YamlFixes + +_TESTS_SECTION = [ + 'name', + 'config', + 'tests', + 'PICS', +] + +_TEST_SECTION = [ + 'label', + 'cluster', + 'command', + 'disabled', + 'endpoint', + 'identity', + 'fabricFiltered', + 'verification', + 'nodeId', + 'attribute', + 'optional', + 'PICS', + 'arguments', + 'response', + 'minInterval', + 'maxInterval', + 'timedInteractionTimeoutMs', + 'busyWaitMs', +] + +_TEST_ARGUMENTS_SECTION = [ + 'values', + 'value', +] + +_TEST_RESPONSE_SECTION = [ + 'value', + 'values', + 'error', + 'constraints', + 'type', + 'hasMasksSet', + 'contains', + 'saveAs' +] + +_ATTRIBUTE_COMMANDS = [ + 'readAttribute', + 'writeAttribute', + 'subscribeAttribute', +] + +_EVENT_COMMANDS = [ + 'readEvent', + 'subscribeEvent', +] + + +# Each 'check' add an entry into the logs db. This entry contains the success of failure state as well as a log +# message describing the check itself. +# A check state can be any of the three valid state: +# * success: Indicates that the check was successfull +# * failure: Indicates that the check was unsuccessfull +# * warning: Indicates that the check is probably successful but that something needs to be considered. +class YamlLog: + def __init__(self, state, category, message): + if not state == 'success' and not state == 'warning' and not state == 'error': + raise ValueError + + self.state = state + self.category = category + self.message = message + + def is_success(self): + return self.state == 'success' + + def is_warning(self): + return self.state == 'warning' + + def is_error(self): + return self.state == 'error' + + +class YamlLogger: + def __init__(self): + self.entries = [] + self.successes = 0 + self.warnings = 0 + self.errors = 0 + + def success(self, category, message): + self.__insert('success', category, message) + self.successes += 1 + pass + + def warning(self, category, message): + self.__insert('warning', category, message) + self.warnings += 1 + pass + + def error(self, category, message): + self.__insert('error', category, message) + self.errors += 1 + pass + + def is_success(self): + return self.errors == 0 + + def is_failure(self): + return self.errors != 0 + + def __insert(self, state, category, message): + log = YamlLog(state, category, message) + self.entries.append(log) + + +def _check_valid_keys(section, valid_keys_dict): + if section: + for key in section: + if not key in valid_keys_dict: + print(f'Unknown key: {key}') + raise KeyError + + +def _valueOrNone(data, key): + return data[key] if key in data else None + + +def _valueOrConfig(data, key, config): + return data[key] if key in data else config[key] + + +class YamlTest: + isEnabled: True + isCommand: False + isAttribute: False + isEvent: False + + def __init__(self, test, config, definitions): + # disabled tests are not parsed in order to allow the test to be added to the test + # suite even if the feature is not implemented yet. + self.isEnabled = not ('disabled' in test and test['disabled']) + if not self.isEnabled: + return + + _check_valid_keys(test, _TEST_SECTION) + + self.label = _valueOrNone(test, 'label') + self.optional = _valueOrNone(test, 'optional') + self.nodeId = _valueOrConfig(test, 'nodeId', config) + self.cluster = _valueOrConfig(test, 'cluster', config) + self.command = _valueOrConfig(test, 'command', config) + self.attribute = _valueOrNone(test, 'attribute') + self.endpoint = _valueOrConfig(test, 'endpoint', config) + + self.identity = _valueOrNone(test, 'identity') + self.fabricFiltered = _valueOrNone(test, 'fabricFiltered') + self.minInterval = _valueOrNone(test, 'minInterval') + self.maxInterval = _valueOrNone(test, 'maxInterval') + self.timedInteractionTimeoutMs = _valueOrNone(test, 'timedInteractionTimeoutMs') + self.busyWaitMs = _valueOrNone(test, 'busyWaitMs') + + self.isAttribute = self.command in _ATTRIBUTE_COMMANDS + self.isEvent = self.command in _EVENT_COMMANDS + + self.arguments = _valueOrNone(test, 'arguments') + self.response = _valueOrNone(test, 'response') + + _check_valid_keys(self.arguments, _TEST_ARGUMENTS_SECTION) + _check_valid_keys(self.response, _TEST_RESPONSE_SECTION) + + if self.isAttribute: + attribute_definition = definitions.get_attribute_definition(self.attribute) + self.arguments = self.__update_with_definition(self.arguments, attribute_definition, config) + self.response = self.__update_with_definition(self.response, attribute_definition, config) + else: + command_definition = definitions.get_command_args_definition(self.command) + response_definition = definitions.get_response_args_definition(self.command) + self.arguments = self.__update_with_definition(self.arguments, command_definition, config) + self.response = self.__update_with_definition(self.response, response_definition, config) + + def check_response(self, response, config): + logger = YamlLogger() + + if (not self.__maybe_check_optional(response, logger)): + return logger + + self.__maybe_check_error(response, logger) + self.__maybe_check_cluster_error(response, logger) + self.__maybe_check_values(response, config, logger) + self.__maybe_check_constraints(response, config, logger) + self.__maybe_save_as(response, config, logger) + + return logger + + def __maybe_check_optional(self, response, logger): + if not self.optional or not 'error' in response: + return True + + error = response['error'] + if error == 'UNSUPPORTED_ATTRIBUTE' or error == 'UNSUPPORTED_COMMAND': + # logger.warning('Optional', f'The response contains the error: "{error}".') + return False + + return True + + def __maybe_check_error(self, response, logger): + if self.response and 'error' in self.response: + expectedError = self.response['error'] + if 'error' in response: + receivedError = response['error'] + if expectedError == receivedError: + logger.success('Error', f'The test expects the "{expectedError}" error which occured successfully.') + else: + logger.error('Error', f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') + else: + logger.error('Error', f'The test expects the "{expectedError}" error but no error occured.') + elif not self.response or not 'error' in self.response: + if 'error' in response: + receivedError = response['error'] + logger.error('Error', f'The test expects no error but the "{receivedError}" error occured.') + # Handle generic success/errors + elif response == 'failure': + receivedError = response + logger.error('Error', f'The test expects no error but the "{receivedError}" error occured.') + else: + logger.success('Error', f'The test expects no error an no error occured.') + + def __maybe_check_cluster_error(self, response, logger): + if self.response and 'clusterError' in self.response: + expectedError = self.response['clusterError'] + if 'clusterError' in response: + receivedError = response['clusterError'] + if expectedError == receivedError: + logger.success('Error', f'The test expects the "{expectedError}" error which occured successfully.') + else: + logger.error('Error', f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') + else: + logger.error('Error', f'The test expects the "{expectedError}" error but no error occured.') + + def __maybe_check_values(self, response, config, logger): + if not self.response or not 'values' in self.response: + return + + if not 'value' in response: + logger.error('Response', f'The test expects some values but none was received.') + return + + expected_entries = self.response['values'] + received_entry = response['value'] + + for expected_entry in expected_entries: + if type(expected_entry) is dict and type(received_entry) is dict: + if not 'value' in expected_entry: + continue + + expected_name = expected_entry['name'] + expected_value = expected_entry['value'] + if not expected_name in received_entry: + if expected_value is None: + logger.success('Response', f'The test expectation "{expected_name} == {expected_value}" is true') + else: + logger.error('Response', f'The test expects a value named "{expected_name}" but it does not exists in the response."') + return + + received_value = received_entry[expected_name] + self.__check_value(expected_name, expected_value, received_value, logger) + else: + if not 'value' in expected_entries[0]: + continue + + expected_value = expected_entry['value'] + received_value = received_entry + self.__check_value('value', expected_value, received_value, logger) + + def __check_value(self, name, expected_value, received_value, logger): + # TODO Supports Array/List. See an exemple of failure in TestArmFailSafe.yaml + if expected_value == received_value: + logger.success('Response', f'The test expectation "{name} == {expected_value}" is true') + else: + logger.error('Response', f'The test expectation "{name} == {expected_value}" is false') + + def __maybe_check_constraints(self, response, config, logger): + if not self.response or not 'constraints' in self.response: + return + constraints = self.response['constraints'] + + allowed_constraint_types = { + 'minValue': { + 'types': [int, float], + 'is_allowed_null': True, + }, + 'maxValue': { + 'types': [int, float], + 'is_allowed_null': True, + }, + 'notValue': { + 'types': '*', + 'is_allowed_null': True, + }, + 'minLength': { + 'types': [str, bytes, list], + 'is_allowed_null': False, + }, + 'maxLength': { + 'types': [str, bytes, list], + 'is_allowed_null': False, + }, + 'isLowerCase': { + 'types': [str], + 'is_allowed_null': False, + }, + 'isUpperCase': { + 'types': [str], + 'is_allowed_null': False, + }, + 'isHexString': { + 'types': [str], + 'is_allowed_null': False, + }, + 'startsWith': { + 'types': [str], + 'is_allowed_null': False, + }, + 'endsWith': { + 'types': [str], + 'is_allowed_null': False, + }, + 'hasMasksSet': { + 'types': [int], + 'is_allowed_null': False, + }, + 'hasMasksClear': { + 'types': [int], + 'is_allowed_null': False, + }, + 'contains': { + 'types': [list], + 'is_allowed_null': False, + }, + 'excludes': { + 'types': [list], + 'is_allowed_null': False, + }, + 'type': { + 'types': '*', + 'is_allowed_null': True, + } + } + + def check_is_valid_type(constraint, value): + allowed = allowed_constraint_types[constraint] + + if value is None and allowed['is_allowed_null']: + return True + + if allowed['types'] == '*' or type(value) in allowed['types']: + return True + + logger.error('Constraints', f'The "{constraint}" constraint is used but the value is not of types {allowed["types"]}.') + return False + + received_value = response['value'] + + for constraint, constraint_value in constraints.items(): + if type(constraint_value) is str and constraint_value in config: + constraint_value = config[constraint_value] + + if not check_is_valid_type(constraint, received_value): + continue + + success = False + warning = False + message = None + + if constraint == 'minValue': + if received_value is None: + continue + + success = received_value >= constraint_value + message = f'{received_value} >= {constraint_value}' + elif constraint == 'maxValue': + if received_value is None: + continue + + success = received_value <= constraint_value + message = f'{received_value} <= {constraint_value}' + elif constraint == 'notValue': + success = received_value != constraint_value + message = f'{received_value} != {constraint_value}' + elif constraint == 'minLength': + success = len(received_value) >= constraint_value + message = f'len({received_value}) >= {constraint_value}' + elif constraint == 'maxLength': + success = len(received_value) <= constraint_value + message = f'len({received_value}) <= {constraint_value}' + elif constraint == 'startsWith': + success = received_value.startswith(constraint_value) + message = f'{received_value} starts with {constraint_value}' + elif constraint == 'endsWith': + success = received_value.endswith(constraint_value) + message = f'{received_value} ends with {constraint_value}' + elif constraint == 'isLowerCase': + success = (received_value == received_value.lower()) == constraint_value + message = f'{received_value} isLowerCase = {constraint_value}' + elif constraint == 'isUpperCase': + success = (received_value == received_value.upper()) == constraint_value + message = f'{received_value} isUpperCase = {constraint_value}' + elif constraint == 'isHexString': + success = all(c in string.hexdigits for c in received_value) == constraint_value + message = f'{received_value} isHexStringCase = {constraint_value}' + elif constraint == 'contains': + success = set(constraint_value).issubset(set(received_value)) + message = f'contains {constraint_value}' + elif constraint == 'excludes': + success = not bool(set(constraint_value) & set(received_value)) + message = f'excludes {constraint_value}' + elif constraint == 'hasMasksSet': + success = True + for mask in constraint_value: + if not received_value & mask: + success = False + message = f'hasMasksSet = {constraint_value}' + elif constraint == 'hasMasksClear': + success = True + for mask in constraint_value: + if received_value & mask: + success = False + message = f'hasMasksClear = {constraint_value}' + elif constraint == 'type': + if constraint_value == 'boolean' and type(received_value) is bool: + success = True + if constraint_value == 'list' and type(received_value) is list: + success = True + elif constraint_value == 'char_string' and type(received_value) is str: + success = True + elif constraint_value == 'octet_string' and type(received_value) is bytes: + success = True + elif constraint_value == 'vendor_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFF + elif constraint_value == 'device_type_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'cluster_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'attribute_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'field_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'command_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'event_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'action_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFF + elif constraint_value == 'transaction_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'node_id' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF + elif constraint_value == 'bitmap8' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFF + elif constraint_value == 'bitmap16' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFF + elif constraint_value == 'bitmap32' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'bitmap64' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF + elif constraint_value == 'enum8' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFF + elif constraint_value == 'enum16' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFF + elif constraint_value == 'Percent' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFF + elif constraint_value == 'Percent100ths' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFF + elif constraint_value == 'epoch_us' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF + elif constraint_value == 'epoch_s' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'utc' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'date' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'tod' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'int8u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFF + elif constraint_value == 'int16u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFF + elif constraint_value == 'int24u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFF + elif constraint_value == 'int32u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFF + elif constraint_value == 'int40u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFF + elif constraint_value == 'int48u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFF + elif constraint_value == 'int56u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFF + elif constraint_value == 'int64u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF + elif constraint_value == 'nullable_int8u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFE + elif constraint_value == 'nullable_int16u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFE + elif constraint_value == 'nullable_int24u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFE + elif constraint_value == 'nullable_int32u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFE + elif constraint_value == 'nullable_int40u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFE + elif constraint_value == 'nullable_int48u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFE + elif constraint_value == 'nullable_int56u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFE + elif constraint_value == 'nullable_int64u' and type(received_value) is int: + success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFE + elif constraint_value == 'int8s' and type(received_value) is int: + success = received_value >= -128 and received_value <= 127 + elif constraint_value == 'int16s' and type(received_value) is int: + success = received_value >= -32768 and received_value <= 32767 + elif constraint_value == 'int24s' and type(received_value) is int: + success = received_value >= -8388608 and received_value <= 8388607 + elif constraint_value == 'int32s' and type(received_value) is int: + success = received_value >= -2147483648 and received_value <= 2147483647 + elif constraint_value == 'int40s' and type(received_value) is int: + success = received_value >= -549755813888 and received_value <= 549755813887 + elif constraint_value == 'int48s' and type(received_value) is int: + success = received_value >= -140737488355328 and received_value <= 140737488355327 + elif constraint_value == 'int56s' and type(received_value) is int: + success = received_value >= -36028797018963968 and received_value <= 36028797018963967 + elif constraint_value == 'int64s' and type(received_value) is int: + success = received_value >= -9223372036854775808 and received_value <= 9223372036854775807 + elif constraint_value == 'nullable_int8s' and type(received_value) is int: + success = received_value >= -127 and received_value <= 127 + elif constraint_value == 'nullable_int16s' and type(received_value) is int: + success = received_value >= -32767 and received_value <= 32767 + elif constraint_value == 'nullable_int24s' and type(received_value) is int: + success = received_value >= -8388607 and received_value <= 8388607 + elif constraint_value == 'nullable_int32s' and type(received_value) is int: + success = received_value >= -2147483647 and received_value <= 2147483647 + elif constraint_value == 'nullable_int40s' and type(received_value) is int: + success = received_value >= -549755813887 and received_value <= 549755813887 + elif constraint_value == 'nullable_int48s' and type(received_value) is int: + success = received_value >= -140737488355327 and received_value <= 140737488355327 + elif constraint_value == 'nullable_int56s' and type(received_value) is int: + success = received_value >= -36028797018963967 and received_value <= 36028797018963967 + elif constraint_value == 'nullable_int64s' and type(received_value) is int: + success = received_value >= -9223372036854775807 and received_value <= 9223372036854775807 + else: + warning = True + message = f'{constraint} == {constraint_value}' + else: + message = f'Unknown constraint type: {constraint}' + + if success: + logger.success('Constraints', f'The test constraint "{message}" succeeds.') + elif warning: + logger.warning('Constraints', f'The test constraints "{message}" is ignored.') + else: + logger.error('Constraints', f'The test constraint "{message}" failed.') + + def __maybe_save_as(self, response, config, logger): + if not self.response or not 'values' in self.response: + return + + if not 'value' in response: + logger.error('SaveAs', f'SaveAs: The test expects some values but none was received.') + return + + expected_entries = self.response['values'] + received_entry = response['value'] + + for expected_entry in expected_entries: + if not 'saveAs' in expected_entry: + continue + saveAs = expected_entry['saveAs'] + + if type(expected_entry) is dict and type(received_entry) is dict: + expected_name = expected_entry['name'] + if not expected_name in received_entry: + logger.error('SaveAs', f'The test expects a value named "{expected_name}" but it does not exists in the response."') + continue + + received_value = received_entry[expected_name] + config[saveAs] = received_value + logger.success('SaveAs', f'The test save the value "{received_value}" as {saveAs}.') + else: + received_value = received_entry + config[saveAs] = received_value + logger.success('SaveAs', f'The test save the value "{received_value}" as {saveAs}.') + + def __update_with_definition(self, container, mapping_type, config): + if not container or not mapping_type: + return container + + for key, items in container.items(): + if type(items) is list and key == 'values': + rv = [] + for item in items: + newItem = {} + for item_key in item: + if item_key == 'value': + newItem[item_key] = self.__update_value_with_definition( + item['value'], mapping_type[item['name']], config) + else: + if item_key == 'saveAs' and not item[item_key] in config: + config[item[item_key]] = None + newItem[item_key] = item[item_key] + rv.append(newItem) + container[key] = rv + elif key == 'value' or key == 'values': + if 'saveAs' in container and not container['saveAs'] in config: + config[container['saveAs']] = None + rv = self.__update_value_with_definition(items, mapping_type, config) + container[key] = rv + elif key == 'constraints': + for constraint in container[key]: + container[key][constraint] = self.__update_value_with_definition( + container[key][constraint], mapping_type, config) + return container + + def __update_value_with_definition(self, value, mapping_type, config): + if not mapping_type: + return value + + if type(value) is dict: + rv = {} + for key in value: + # FabricIndex is a special case where the framework requires it to be passed even if it is not part of the + # requested arguments per spec and not part of the XML definition. + if key == 'FabricIndex' or key == 'fabricIndex': + rv[key] = value[key] # int64u + else: + mapping = mapping_type[key] + rv[key] = self.__update_value_with_definition(value[key], mapping, config) + value = rv + elif type(value) is list: + value = [self.__update_value_with_definition(entry, mapping_type, config) for entry in value] + elif value and not value in config: + if mapping_type == 'int64u' or mapping_type == 'int64s' or mapping_type == 'bitmap64' or mapping_type == 'epoch_us': + value = YamlFixes.try_apply_yaml_cpp_longlong_limitation_fix(value) + value = YamlFixes.try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value) + elif mapping_type == 'single' or mapping_type == 'double': + value = YamlFixes.try_apply_yaml_float_written_as_strings(value) + elif mapping_type == 'octet_string' or mapping_type == 'long_octet_string': + if value.startswith('hex:'): + value = bytes.fromhex(value[4:]) + else: + value = value.encode() + elif mapping_type == 'boolean': + value = bool(value) + + return value + + +class YamlTests: + __tests: None + __hook: None + __index: 0 + + count: 0 + + def __init__(self, tests, hook): + self.__tests = tests + self.__hook = hook + self.__index = 0 + + self.count = len(tests) + + def __iter__(self): + return self + + def __next__(self): + if self.__index < self.count: + data = self.__tests[self.__index] + data = self.__hook(data) + self.__index += 1 + return data + + raise StopIteration + + +class YamlParser: + name: None + PICS: None + config: None + tests: None + + def __init__(self, test_file, pics_file, definitions): + # TODO Needs supports for PICS file + with open(test_file) as f: + loader = yaml.FullLoader + loader = YamlFixes.try_add_yaml_support_for_scientific_notation_without_dot(loader) + + data = yaml.load(f, Loader=loader) + _check_valid_keys(data, _TESTS_SECTION) + + self.name = _valueOrNone(data, 'name') + self.PICS = _valueOrNone(data, 'PICS') + self.config = _valueOrNone(data, 'config') + + tests = list(filter(lambda test: test.isEnabled, [YamlTest( + test, self.config, definitions) for test in _valueOrNone(data, 'tests')])) + YamlFixes.try_update_yaml_node_id_test_runner_state(tests, self.config) + + self.tests = YamlTests(tests, self.__updatePlaceholder) + + def __updatePlaceholder(self, data): + data.arguments = self.__encode_values(data.arguments) + data.response = self.__encode_values(data.response) + return data + + def __encode_values(self, container): + if not container: + return None + + if 'value' in container: + container['values'] = [{'name': 'value', 'value': container['value']}] + del container['value'] + + if 'values' in container: + for idx, item in enumerate(container['values']): + if 'value' in item: + container['values'][idx]['value'] = self.__encode_value_with_config(item['value'], self.config) + + if 'saveAs' in container: + if not 'values' in container: + container['values'] = [{}] + container['values'][0]['saveAs'] = container['saveAs'] + del container['saveAs'] + + return container + + def __encode_value_with_config(self, value, config): + if type(value) is list: + return [self.__encode_value_with_config(entry, config) for entry in value] + elif type(value) is dict: + mapped_value = {} + for key in value: + mapped_value[key] = self.__encode_value_with_config(value[key], config) + return mapped_value + elif type(value) is str: + # For most tests, a single config variable is used and it can be replaced as in. + # But some other tests were relying on the fact that the expression was put 'as if' in + # the generated code and was resolved before beeing sent over the wire. For such expressions + # (e.g 'myVar + 1') we need to compute it before sending it over the wire. + tokens = value.split() + if len(tokens) == 0: + return value + + has_items_from_config = False + for idx, token in enumerate(tokens): + if token in config: + config_item = self.config[token] + if type(config_item) is dict and 'defaultValue' in config_item: + config_item = config_item['defaultValue'] + tokens[idx] = config_item + has_items_from_config = True + + if len(tokens) == 1: + return tokens[0] + + tokens = [str(token) for token in tokens] + value = ' '.join(tokens) + return value if not has_items_from_config else eval(value) + else: + return value + + def updateConfig(key, value): + self.config[key] = value From 011180ebbca980d0c02fe2c1e9b4683d459bf131 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 2 Dec 2022 09:50:19 -0500 Subject: [PATCH 02/23] Adding a test commit containing a TODO --- scripts/tests/yamltests/ClustersDefinitions.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/tests/yamltests/ClustersDefinitions.py b/scripts/tests/yamltests/ClustersDefinitions.py index f08eabbeee581f..db05922abd2117 100644 --- a/scripts/tests/yamltests/ClustersDefinitions.py +++ b/scripts/tests/yamltests/ClustersDefinitions.py @@ -18,6 +18,8 @@ import time +# TODO(use scripts/idl/xml_parser.py for zap xml parsing) + class ClustersDefinitions: def __init__(self, clusters_dir): self.__clusterIdToName = {} @@ -91,7 +93,8 @@ def __init__(self, clusters_dir): command_code = command.get('code') command_name = command.get('name') - base = 16 if command_code.startswith('0x') or command_code.startswith('0X') else 10 + base = 16 if command_code.startswith( + '0x') or command_code.startswith('0X') else 10 command_code = int(command_code, base=base) self.__commandIdToName[cluster_code][command_code] = command_name @@ -109,7 +112,8 @@ def __init__(self, clusters_dir): description = attribute.find('description') attribute_name = description.text if description is not None else attribute.text - base = 16 if attribute_code.startswith('0x') or attribute_code.startswith('0X') else 10 + base = 16 if attribute_code.startswith( + '0x') or attribute_code.startswith('0X') else 10 attribute_code = int(attribute_code, base=base) self.__attributeIdToName[cluster_code][attribute_code] = attribute_name @@ -124,7 +128,8 @@ def __init__(self, clusters_dir): description = event.find('description') event_name = description.text if description is not None else event.text - base = 16 if event_code.startswith('0x') or event_code.startswith('0X') else 10 + base = 16 if event_code.startswith( + '0x') or event_code.startswith('0X') else 10 event_code = int(event_code, base=base) self.__eventIdToName[cluster_code][event_code] = event_name @@ -154,7 +159,8 @@ def get_response_mapping(self, command_name): mapping = {} for mapping_index, arg in enumerate(args): - mapping[str(mapping_index)] = {'name': arg.get('name'), 'type': arg.get('type').lower()} + mapping[str(mapping_index)] = {'name': arg.get( + 'name'), 'type': arg.get('type').lower()} return mapping def get_attribute_mapping(self, attribute_name): @@ -192,7 +198,8 @@ def get_type_mapping(self, type_name): for item in items: if item.get('fieldId'): mapping_index = int(item.get('fieldId')) - mapping[str(mapping_index)] = {'name': item.get('name'), 'type': item.get('type').lower()} + mapping[str(mapping_index)] = {'name': item.get( + 'name'), 'type': item.get('type').lower()} mapping_index += 1 return mapping From 1ef84a6ac5c05df9f74c773c722ff18b5e099d28 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 2 Dec 2022 09:51:43 -0500 Subject: [PATCH 03/23] Restyle --- scripts/tests/yamltests/YamlParser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index e4a8c8a72a407b..af646c0994d380 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -278,7 +278,8 @@ def __maybe_check_values(self, response, config, logger): if expected_value is None: logger.success('Response', f'The test expectation "{expected_name} == {expected_value}" is true') else: - logger.error('Response', f'The test expects a value named "{expected_name}" but it does not exists in the response."') + logger.error( + 'Response', f'The test expects a value named "{expected_name}" but it does not exists in the response."') return received_value = received_entry[expected_name] @@ -596,7 +597,8 @@ def __maybe_save_as(self, response, config, logger): if type(expected_entry) is dict and type(received_entry) is dict: expected_name = expected_entry['name'] if not expected_name in received_entry: - logger.error('SaveAs', f'The test expects a value named "{expected_name}" but it does not exists in the response."') + logger.error( + 'SaveAs', f'The test expects a value named "{expected_name}" but it does not exists in the response."') continue received_value = received_entry[expected_name] From 4fcd6a196b442aeb52cdf752d645363a4272a8d5 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Mon, 5 Dec 2022 15:10:14 +0000 Subject: [PATCH 04/23] Small tweaks plus adding unit test * Moving towards PEP8 naming convension * Refactor results to be less focused string matching * Added very basic unit test file Test run using: `python3 scripts/tests/test_yaml_parser.py` --- scripts/tests/test_yaml_parser.py | 80 +++++++++ scripts/tests/yamltests/YamlFixes.py | 2 +- scripts/tests/yamltests/YamlParser.py | 232 +++++++++++++++----------- 3 files changed, 215 insertions(+), 99 deletions(-) create mode 100644 scripts/tests/test_yaml_parser.py diff --git a/scripts/tests/test_yaml_parser.py b/scripts/tests/test_yaml_parser.py new file mode 100644 index 00000000000000..f864c6dadf973a --- /dev/null +++ b/scripts/tests/test_yaml_parser.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2022 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# TODO Once yamltest is a proper self contained module we can move this file +# to a more appropriate spot. For now, having this file to do some quick checks +# is arguably better then no checks at all. + +import os +import unittest +from pathlib import Path + +from yamltests.ClustersDefinitions import ClustersDefinitions +from yamltests.YamlParser import YamlParser + +_DEFAULT_MATTER_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) +_YAML_TEST_SUITE_PATH = os.path.abspath( + os.path.join(_DEFAULT_MATTER_ROOT, 'src/app/tests/suites')) + +_CLUSTER_DEFINITION_DIRECTORY = os.path.abspath( + os.path.join(_DEFAULT_MATTER_ROOT, 'src/app/zap-templates/zcl/data-model')) + + +class TestYamlParser(unittest.TestCase): + def setUp(self): + # TODO we should not be reliant on an external YAML file. Possible that we should have + # either a local .yaml testfile, of the file contents should exist in this file where we + # write out the yaml file to temp directory for our testing use. This approach was taken + # since this test is better than no test. + yaml_test_suite_path = Path(_YAML_TEST_SUITE_PATH) + if not yaml_test_suite_path.exists(): + raise FileNotFoundError(f'Expected directory {_YAML_TEST_SUITE_PATH} to exist') + yaml_test_filename = 'TestCluster.yaml' + path_to_test = None + for path in yaml_test_suite_path.rglob(yaml_test_filename): + if not path.is_file(): + continue + if path.name != yaml_test_filename: + continue + path_to_test = str(path) + break + if path_to_test is None: + raise FileNotFoundError(f'Could not file {yaml_test_filename} in directory {_YAML_TEST_SUITE_PATH}') + pics_file = None + + # TODO Again we should not be reliant on extneral XML files. But some test (even brittal) + # are better than no tests. + + clusters_definitions = ClustersDefinitions(_CLUSTER_DEFINITION_DIRECTORY) + + self._yaml_parser = YamlParser(path_to_test, pics_file, clusters_definitions) + + def test_able_to_iterate_over_all_tests(self): + # self._yaml_parser.tests implements `__next__`, which does value substitution. We are + # simply ensure there is no exceptions raise. + for idx, test_step in enumerate(self._yaml_parser.tests): + pass + self.assertTrue(True) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/scripts/tests/yamltests/YamlFixes.py b/scripts/tests/yamltests/YamlFixes.py index 1178090815574b..019d1450417d66 100644 --- a/scripts/tests/yamltests/YamlFixes.py +++ b/scripts/tests/yamltests/YamlFixes.py @@ -69,7 +69,7 @@ def try_update_yaml_node_id_test_runner_state(tests, config): identities = {'alpha': None if 'nodeId' not in config else config['nodeId']} for test in tests: - if not test.isEnabled: + if not test.is_enabled: continue identity = test.identity diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index af646c0994d380..8adc8f594c9def 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -15,6 +15,7 @@ import yaml import string +from enum import Enum from . import YamlFixes @@ -74,61 +75,81 @@ ] +class PostProcessCheckStatus(Enum): + SUCCESS = 'success', + WARNING = 'warning', + ERROR = 'error' + + +# TODO these values are just what was there before, these should be updated +class PostProcessCheckType(Enum): + IM_STATUS = 'Error', + CLUSTER_STATUS = 'ClusterError', + RESPONSE_VALIDATION = 'Response', + CONSTRAINT_VALIDATION = 'Constraints', + SAVE_AS_VARIABLE = 'SaveAs' + + # Each 'check' add an entry into the logs db. This entry contains the success of failure state as well as a log # message describing the check itself. # A check state can be any of the three valid state: # * success: Indicates that the check was successfull # * failure: Indicates that the check was unsuccessfull # * warning: Indicates that the check is probably successful but that something needs to be considered. -class YamlLog: - def __init__(self, state, category, message): - if not state == 'success' and not state == 'warning' and not state == 'error': - raise ValueError - +class PostProcessCheck: + def __init__(self, state: PostProcessCheckStatus, category: PostProcessCheckType, message: str): self.state = state self.category = category self.message = message - def is_success(self): - return self.state == 'success' + def is_success(self) -> bool: + return self.state == PostProcessCheckStatus.SUCCESS + + def is_warning(self) -> bool: + return self.state == PostProcessCheckStatus.WARNING + + def is_error(self) -> bool: + return self.state == PostProcessCheckStatus.ERROR - def is_warning(self): - return self.state == 'warning' - def is_error(self): - return self.state == 'error' +class PostProcessResponseResult: + ''' asdf + There are multiple steps that occur when post processing a response. This is a summary of the + results. Note that the number and types of steps performed is dependant on test step itself. + ''' -class YamlLogger: def __init__(self): self.entries = [] self.successes = 0 self.warnings = 0 self.errors = 0 - def success(self, category, message): - self.__insert('success', category, message) + def success(self, category: PostProcessCheckType, message: str): + ''' Adds a success entry that occured when post processing response to results.''' + self.__insert(PostProcessCheckStatus.SUCCESS, category, message) self.successes += 1 - pass - def warning(self, category, message): - self.__insert('warning', category, message) + def warning(self, category: PostProcessCheckType, message: str): + ''' Adds a warning entry that occured when post processing response to results.''' + self.__insert(PostProcessCheckStatus.WARNING, category, message) self.warnings += 1 - pass - def error(self, category, message): - self.__insert('error', category, message) + def error(self, category: PostProcessCheckType, message: str): + ''' Adds an error entry that occured when post processing response to results.''' + self.__insert(PostProcessCheckStatus.ERROR, category, message) self.errors += 1 - pass def is_success(self): + # It is possible that post processing a response doesn't have any success entires added + # that is why we explicitly only search for if an error occurred. return self.errors == 0 def is_failure(self): return self.errors != 0 - def __insert(self, state, category, message): - log = YamlLog(state, category, message) + def __insert(self, state: PostProcessCheckStatus, category: PostProcessCheckType, message: str): + log = PostProcessCheck(state, category, message) self.entries.append(log) @@ -148,17 +169,17 @@ def _valueOrConfig(data, key, config): return data[key] if key in data else config[key] -class YamlTest: - isEnabled: True - isCommand: False - isAttribute: False - isEvent: False +class TestStep: + is_enabled: True + is_command: False + is_attribute: False + is_event: False - def __init__(self, test, config, definitions): - # disabled tests are not parsed in order to allow the test to be added to the test + def __init__(self, test: dict, config: dict, definitions): + # Disabled tests are not parsed in order to allow the test to be added to the test # suite even if the feature is not implemented yet. - self.isEnabled = not ('disabled' in test and test['disabled']) - if not self.isEnabled: + self.is_enabled = not ('disabled' in test and test['disabled']) + if not self.is_enabled: return _check_valid_keys(test, _TEST_SECTION) @@ -178,8 +199,8 @@ def __init__(self, test, config, definitions): self.timedInteractionTimeoutMs = _valueOrNone(test, 'timedInteractionTimeoutMs') self.busyWaitMs = _valueOrNone(test, 'busyWaitMs') - self.isAttribute = self.command in _ATTRIBUTE_COMMANDS - self.isEvent = self.command in _EVENT_COMMANDS + self.is_attribute = self.command in _ATTRIBUTE_COMMANDS + self.is_event = self.command in _EVENT_COMMANDS self.arguments = _valueOrNone(test, 'arguments') self.response = _valueOrNone(test, 'response') @@ -187,7 +208,7 @@ def __init__(self, test, config, definitions): _check_valid_keys(self.arguments, _TEST_ARGUMENTS_SECTION) _check_valid_keys(self.response, _TEST_RESPONSE_SECTION) - if self.isAttribute: + if self.is_attribute: attribute_definition = definitions.get_attribute_definition(self.attribute) self.arguments = self.__update_with_definition(self.arguments, attribute_definition, config) self.response = self.__update_with_definition(self.response, attribute_definition, config) @@ -197,71 +218,76 @@ def __init__(self, test, config, definitions): self.arguments = self.__update_with_definition(self.arguments, command_definition, config) self.response = self.__update_with_definition(self.response, response_definition, config) - def check_response(self, response, config): - logger = YamlLogger() + def post_process_response(self, response, config): + result = PostProcessResponseResult() - if (not self.__maybe_check_optional(response, logger)): - return logger + if (not self.__maybe_check_optional(response, result)): + return result - self.__maybe_check_error(response, logger) - self.__maybe_check_cluster_error(response, logger) - self.__maybe_check_values(response, config, logger) - self.__maybe_check_constraints(response, config, logger) - self.__maybe_save_as(response, config, logger) + self.__maybe_check_error(response, result) + self.__maybe_check_cluster_error(response, result) + self.__maybe_check_values(response, config, result) + self.__maybe_check_constraints(response, config, result) + self.__maybe_save_as(response, config, result) - return logger + return result - def __maybe_check_optional(self, response, logger): + def __maybe_check_optional(self, response, result): if not self.optional or not 'error' in response: return True error = response['error'] if error == 'UNSUPPORTED_ATTRIBUTE' or error == 'UNSUPPORTED_COMMAND': - # logger.warning('Optional', f'The response contains the error: "{error}".') + # result.warning('Optional', f'The response contains the error: "{error}".') return False return True - def __maybe_check_error(self, response, logger): + def __maybe_check_error(self, response, result): if self.response and 'error' in self.response: expectedError = self.response['error'] if 'error' in response: receivedError = response['error'] if expectedError == receivedError: - logger.success('Error', f'The test expects the "{expectedError}" error which occured successfully.') + result.success(PostProcessCheckType.IM_STATUS, + f'The test expects the "{expectedError}" error which occured successfully.') else: - logger.error('Error', f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') + result.error(PostProcessCheckType.IM_STATUS, + f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') else: - logger.error('Error', f'The test expects the "{expectedError}" error but no error occured.') + result.error(PostProcessCheckType.IM_STATUS, f'The test expects the "{expectedError}" error but no error occured.') elif not self.response or not 'error' in self.response: if 'error' in response: receivedError = response['error'] - logger.error('Error', f'The test expects no error but the "{receivedError}" error occured.') + result.error(PostProcessCheckType.IM_STATUS, f'The test expects no error but the "{receivedError}" error occured.') # Handle generic success/errors elif response == 'failure': receivedError = response - logger.error('Error', f'The test expects no error but the "{receivedError}" error occured.') + result.error(PostProcessCheckType.IM_STATUS, f'The test expects no error but the "{receivedError}" error occured.') else: - logger.success('Error', f'The test expects no error an no error occured.') + result.success(PostProcessCheckType.IM_STATUS, f'The test expects no error an no error occured.') - def __maybe_check_cluster_error(self, response, logger): + def __maybe_check_cluster_error(self, response, result): if self.response and 'clusterError' in self.response: expectedError = self.response['clusterError'] if 'clusterError' in response: receivedError = response['clusterError'] if expectedError == receivedError: - logger.success('Error', f'The test expects the "{expectedError}" error which occured successfully.') + result.success(PostProcessCheckType.CLUSTER_STATUS, + f'The test expects the "{expectedError}" error which occured successfully.') else: - logger.error('Error', f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') + result.error(PostProcessCheckType.CLUSTER_STATUS, + f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') else: - logger.error('Error', f'The test expects the "{expectedError}" error but no error occured.') + result.error(PostProcessCheckType.CLUSTER_STATUS, + f'The test expects the "{expectedError}" error but no error occured.') - def __maybe_check_values(self, response, config, logger): + def __maybe_check_values(self, response, config, result): if not self.response or not 'values' in self.response: return if not 'value' in response: - logger.error('Response', f'The test expects some values but none was received.') + result.error(PostProcessCheckType.RESPONSE_VALIDATION, f'The test expects some values but none was received.') return expected_entries = self.response['values'] @@ -276,30 +302,31 @@ def __maybe_check_values(self, response, config, logger): expected_value = expected_entry['value'] if not expected_name in received_entry: if expected_value is None: - logger.success('Response', f'The test expectation "{expected_name} == {expected_value}" is true') + result.success(PostProcessCheckType.RESPONSE_VALIDATION, + f'The test expectation "{expected_name} == {expected_value}" is true') else: - logger.error( - 'Response', f'The test expects a value named "{expected_name}" but it does not exists in the response."') + result.error( + PostProcessCheckType.RESPONSE_VALIDATION, f'The test expects a value named "{expected_name}" but it does not exists in the response."') return received_value = received_entry[expected_name] - self.__check_value(expected_name, expected_value, received_value, logger) + self.__check_value(expected_name, expected_value, received_value, result) else: if not 'value' in expected_entries[0]: continue expected_value = expected_entry['value'] received_value = received_entry - self.__check_value('value', expected_value, received_value, logger) + self.__check_value('value', expected_value, received_value, result) - def __check_value(self, name, expected_value, received_value, logger): + def __check_value(self, name, expected_value, received_value, result): # TODO Supports Array/List. See an exemple of failure in TestArmFailSafe.yaml if expected_value == received_value: - logger.success('Response', f'The test expectation "{name} == {expected_value}" is true') + result.success(PostProcessCheckType.RESPONSE_VALIDATION, f'The test expectation "{name} == {expected_value}" is true') else: - logger.error('Response', f'The test expectation "{name} == {expected_value}" is false') + result.error(PostProcessCheckType.RESPONSE_VALIDATION, f'The test expectation "{name} == {expected_value}" is false') - def __maybe_check_constraints(self, response, config, logger): + def __maybe_check_constraints(self, response, config, result): if not self.response or not 'constraints' in self.response: return constraints = self.response['constraints'] @@ -376,7 +403,8 @@ def check_is_valid_type(constraint, value): if allowed['types'] == '*' or type(value) in allowed['types']: return True - logger.error('Constraints', f'The "{constraint}" constraint is used but the value is not of types {allowed["types"]}.') + result.error(PostProcessCheckType.CONSTRAINT_VALIDATION, + f'The "{constraint}" constraint is used but the value is not of types {allowed["types"]}.') return False received_value = response['value'] @@ -572,18 +600,18 @@ def check_is_valid_type(constraint, value): message = f'Unknown constraint type: {constraint}' if success: - logger.success('Constraints', f'The test constraint "{message}" succeeds.') + result.success(PostProcessCheckType.CONSTRAINT_VALIDATION, f'The test constraint "{message}" succeeds.') elif warning: - logger.warning('Constraints', f'The test constraints "{message}" is ignored.') + result.warning(PostProcessCheckType.CONSTRAINT_VALIDATION, f'The test constraints "{message}" is ignored.') else: - logger.error('Constraints', f'The test constraint "{message}" failed.') + result.error(PostProcessCheckType.CONSTRAINT_VALIDATION, f'The test constraint "{message}" failed.') - def __maybe_save_as(self, response, config, logger): - if not self.response or not 'values' in self.response: + def __maybe_save_as(self, response, config, result): + if not self.response or 'values' not in self.response: return if not 'value' in response: - logger.error('SaveAs', f'SaveAs: The test expects some values but none was received.') + result.error(PostProcessCheckType.SAVE_AS_VARIABLE, f'The test expects some values but none was received.') return expected_entries = self.response['values'] @@ -597,17 +625,17 @@ def __maybe_save_as(self, response, config, logger): if type(expected_entry) is dict and type(received_entry) is dict: expected_name = expected_entry['name'] if not expected_name in received_entry: - logger.error( - 'SaveAs', f'The test expects a value named "{expected_name}" but it does not exists in the response."') + result.error( + PostProcessCheckType.SAVE_AS_VARIABLE, f'The test expects a value named "{expected_name}" but it does not exists in the response."') continue received_value = received_entry[expected_name] config[saveAs] = received_value - logger.success('SaveAs', f'The test save the value "{received_value}" as {saveAs}.') + result.success(PostProcessCheckType.SAVE_AS_VARIABLE, f'The test save the value "{received_value}" as {saveAs}.') else: received_value = received_entry config[saveAs] = received_value - logger.success('SaveAs', f'The test save the value "{received_value}" as {saveAs}.') + result.success(PostProcessCheckType.SAVE_AS_VARIABLE, f'The test save the value "{received_value}" as {saveAs}.') def __update_with_definition(self, container, mapping_type, config): if not container or not mapping_type: @@ -703,7 +731,8 @@ def __next__(self): class YamlParser: name: None PICS: None - config: None + # TODO config should be internal (_config). Also can it actually be None, or + config: dict = {} tests: None def __init__(self, test_file, pics_file, definitions): @@ -717,15 +746,16 @@ def __init__(self, test_file, pics_file, definitions): self.name = _valueOrNone(data, 'name') self.PICS = _valueOrNone(data, 'PICS') + self.config = _valueOrNone(data, 'config') - tests = list(filter(lambda test: test.isEnabled, [YamlTest( + tests = list(filter(lambda test: test.is_enabled, [TestStep( test, self.config, definitions) for test in _valueOrNone(data, 'tests')])) YamlFixes.try_update_yaml_node_id_test_runner_state(tests, self.config) - self.tests = YamlTests(tests, self.__updatePlaceholder) + self.tests = YamlTests(tests, self.__update_placeholder) - def __updatePlaceholder(self, data): + def __update_placeholder(self, data): data.arguments = self.__encode_values(data.arguments) data.response = self.__encode_values(data.response) return data @@ -734,6 +764,7 @@ def __encode_values(self, container): if not container: return None + # TODO this should likely be moved to end of TestStep.__init__ if 'value' in container: container['values'] = [{'name': 'value', 'value': container['value']}] del container['value'] @@ -741,50 +772,55 @@ def __encode_values(self, container): if 'values' in container: for idx, item in enumerate(container['values']): if 'value' in item: - container['values'][idx]['value'] = self.__encode_value_with_config(item['value'], self.config) + container['values'][idx]['value'] = self.__config_variable_substitution(item['value']) + # TODO this should likely be moved to end of TestStep.__init__. But depends on rationale if 'saveAs' in container: if not 'values' in container: + # TODO Currently very unclear why this corner case is needed. Would be nice to add + # information as to why this would happen. container['values'] = [{}] container['values'][0]['saveAs'] = container['saveAs'] del container['saveAs'] return container - def __encode_value_with_config(self, value, config): + def __config_variable_substitution(self, value): if type(value) is list: - return [self.__encode_value_with_config(entry, config) for entry in value] + return [self.__config_variable_substitution(entry) for entry in value] elif type(value) is dict: mapped_value = {} for key in value: - mapped_value[key] = self.__encode_value_with_config(value[key], config) + mapped_value[key] = self.__config_variable_substitution(value[key]) return mapped_value elif type(value) is str: # For most tests, a single config variable is used and it can be replaced as in. # But some other tests were relying on the fact that the expression was put 'as if' in - # the generated code and was resolved before beeing sent over the wire. For such expressions + # the generated code and was resolved before being sent over the wire. For such expressions # (e.g 'myVar + 1') we need to compute it before sending it over the wire. tokens = value.split() if len(tokens) == 0: return value - has_items_from_config = False + substitution_occured = False for idx, token in enumerate(tokens): - if token in config: - config_item = self.config[token] - if type(config_item) is dict and 'defaultValue' in config_item: - config_item = config_item['defaultValue'] - tokens[idx] = config_item - has_items_from_config = True + if token in self.config: + variable_info = self.config[token] + if type(variable_info) is dict and 'defaultValue' in variable_info: + variable_info = variable_info['defaultValue'] + tokens[idx] = variable_info + substitution_occured = True if len(tokens) == 1: return tokens[0] tokens = [str(token) for token in tokens] value = ' '.join(tokens) - return value if not has_items_from_config else eval(value) + # TODO we should move away from eval. That will mean that we will need to do extra parsing, + # but it would be safer then just blindly running eval. + return value if not substitution_occured else eval(value) else: return value - def updateConfig(key, value): + def update_config(self, key, value): self.config[key] = value From 1b410ef9c859bb94d13161b960117cfd6fe5ea44 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Tue, 6 Dec 2022 17:43:52 +0000 Subject: [PATCH 05/23] Refactor constraints --- scripts/tests/yamltests/YamlParser.py | 286 +------------------- scripts/tests/yamltests/constraints.py | 357 +++++++++++++++++++++++++ 2 files changed, 369 insertions(+), 274 deletions(-) create mode 100644 scripts/tests/yamltests/constraints.py diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index 8adc8f594c9def..8db1557a7bc688 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -17,6 +17,7 @@ import string from enum import Enum +from .constraints import get_constraints from . import YamlFixes _TESTS_SECTION = [ @@ -329,282 +330,19 @@ def __check_value(self, name, expected_value, received_value, result): def __maybe_check_constraints(self, response, config, result): if not self.response or not 'constraints' in self.response: return - constraints = self.response['constraints'] - - allowed_constraint_types = { - 'minValue': { - 'types': [int, float], - 'is_allowed_null': True, - }, - 'maxValue': { - 'types': [int, float], - 'is_allowed_null': True, - }, - 'notValue': { - 'types': '*', - 'is_allowed_null': True, - }, - 'minLength': { - 'types': [str, bytes, list], - 'is_allowed_null': False, - }, - 'maxLength': { - 'types': [str, bytes, list], - 'is_allowed_null': False, - }, - 'isLowerCase': { - 'types': [str], - 'is_allowed_null': False, - }, - 'isUpperCase': { - 'types': [str], - 'is_allowed_null': False, - }, - 'isHexString': { - 'types': [str], - 'is_allowed_null': False, - }, - 'startsWith': { - 'types': [str], - 'is_allowed_null': False, - }, - 'endsWith': { - 'types': [str], - 'is_allowed_null': False, - }, - 'hasMasksSet': { - 'types': [int], - 'is_allowed_null': False, - }, - 'hasMasksClear': { - 'types': [int], - 'is_allowed_null': False, - }, - 'contains': { - 'types': [list], - 'is_allowed_null': False, - }, - 'excludes': { - 'types': [list], - 'is_allowed_null': False, - }, - 'type': { - 'types': '*', - 'is_allowed_null': True, - } - } - - def check_is_valid_type(constraint, value): - allowed = allowed_constraint_types[constraint] - - if value is None and allowed['is_allowed_null']: - return True - - if allowed['types'] == '*' or type(value) in allowed['types']: - return True - - result.error(PostProcessCheckType.CONSTRAINT_VALIDATION, - f'The "{constraint}" constraint is used but the value is not of types {allowed["types"]}.') - return False - - received_value = response['value'] - - for constraint, constraint_value in constraints.items(): - if type(constraint_value) is str and constraint_value in config: - constraint_value = config[constraint_value] - - if not check_is_valid_type(constraint, received_value): - continue - - success = False - warning = False - message = None - - if constraint == 'minValue': - if received_value is None: - continue - - success = received_value >= constraint_value - message = f'{received_value} >= {constraint_value}' - elif constraint == 'maxValue': - if received_value is None: - continue - success = received_value <= constraint_value - message = f'{received_value} <= {constraint_value}' - elif constraint == 'notValue': - success = received_value != constraint_value - message = f'{received_value} != {constraint_value}' - elif constraint == 'minLength': - success = len(received_value) >= constraint_value - message = f'len({received_value}) >= {constraint_value}' - elif constraint == 'maxLength': - success = len(received_value) <= constraint_value - message = f'len({received_value}) <= {constraint_value}' - elif constraint == 'startsWith': - success = received_value.startswith(constraint_value) - message = f'{received_value} starts with {constraint_value}' - elif constraint == 'endsWith': - success = received_value.endswith(constraint_value) - message = f'{received_value} ends with {constraint_value}' - elif constraint == 'isLowerCase': - success = (received_value == received_value.lower()) == constraint_value - message = f'{received_value} isLowerCase = {constraint_value}' - elif constraint == 'isUpperCase': - success = (received_value == received_value.upper()) == constraint_value - message = f'{received_value} isUpperCase = {constraint_value}' - elif constraint == 'isHexString': - success = all(c in string.hexdigits for c in received_value) == constraint_value - message = f'{received_value} isHexStringCase = {constraint_value}' - elif constraint == 'contains': - success = set(constraint_value).issubset(set(received_value)) - message = f'contains {constraint_value}' - elif constraint == 'excludes': - success = not bool(set(constraint_value) & set(received_value)) - message = f'excludes {constraint_value}' - elif constraint == 'hasMasksSet': - success = True - for mask in constraint_value: - if not received_value & mask: - success = False - message = f'hasMasksSet = {constraint_value}' - elif constraint == 'hasMasksClear': - success = True - for mask in constraint_value: - if received_value & mask: - success = False - message = f'hasMasksClear = {constraint_value}' - elif constraint == 'type': - if constraint_value == 'boolean' and type(received_value) is bool: - success = True - if constraint_value == 'list' and type(received_value) is list: - success = True - elif constraint_value == 'char_string' and type(received_value) is str: - success = True - elif constraint_value == 'octet_string' and type(received_value) is bytes: - success = True - elif constraint_value == 'vendor_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFF - elif constraint_value == 'device_type_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'cluster_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'attribute_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'field_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'command_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'event_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'action_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFF - elif constraint_value == 'transaction_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'node_id' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF - elif constraint_value == 'bitmap8' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFF - elif constraint_value == 'bitmap16' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFF - elif constraint_value == 'bitmap32' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'bitmap64' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF - elif constraint_value == 'enum8' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFF - elif constraint_value == 'enum16' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFF - elif constraint_value == 'Percent' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFF - elif constraint_value == 'Percent100ths' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFF - elif constraint_value == 'epoch_us' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF - elif constraint_value == 'epoch_s' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'utc' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'date' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'tod' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'int8u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFF - elif constraint_value == 'int16u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFF - elif constraint_value == 'int24u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFF - elif constraint_value == 'int32u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFF - elif constraint_value == 'int40u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFF - elif constraint_value == 'int48u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFF - elif constraint_value == 'int56u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFF - elif constraint_value == 'int64u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFF - elif constraint_value == 'nullable_int8u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFE - elif constraint_value == 'nullable_int16u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFE - elif constraint_value == 'nullable_int24u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFE - elif constraint_value == 'nullable_int32u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFE - elif constraint_value == 'nullable_int40u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFE - elif constraint_value == 'nullable_int48u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFE - elif constraint_value == 'nullable_int56u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFE - elif constraint_value == 'nullable_int64u' and type(received_value) is int: - success = received_value >= 0 and received_value <= 0xFFFFFFFFFFFFFFFE - elif constraint_value == 'int8s' and type(received_value) is int: - success = received_value >= -128 and received_value <= 127 - elif constraint_value == 'int16s' and type(received_value) is int: - success = received_value >= -32768 and received_value <= 32767 - elif constraint_value == 'int24s' and type(received_value) is int: - success = received_value >= -8388608 and received_value <= 8388607 - elif constraint_value == 'int32s' and type(received_value) is int: - success = received_value >= -2147483648 and received_value <= 2147483647 - elif constraint_value == 'int40s' and type(received_value) is int: - success = received_value >= -549755813888 and received_value <= 549755813887 - elif constraint_value == 'int48s' and type(received_value) is int: - success = received_value >= -140737488355328 and received_value <= 140737488355327 - elif constraint_value == 'int56s' and type(received_value) is int: - success = received_value >= -36028797018963968 and received_value <= 36028797018963967 - elif constraint_value == 'int64s' and type(received_value) is int: - success = received_value >= -9223372036854775808 and received_value <= 9223372036854775807 - elif constraint_value == 'nullable_int8s' and type(received_value) is int: - success = received_value >= -127 and received_value <= 127 - elif constraint_value == 'nullable_int16s' and type(received_value) is int: - success = received_value >= -32767 and received_value <= 32767 - elif constraint_value == 'nullable_int24s' and type(received_value) is int: - success = received_value >= -8388607 and received_value <= 8388607 - elif constraint_value == 'nullable_int32s' and type(received_value) is int: - success = received_value >= -2147483647 and received_value <= 2147483647 - elif constraint_value == 'nullable_int40s' and type(received_value) is int: - success = received_value >= -549755813887 and received_value <= 549755813887 - elif constraint_value == 'nullable_int48s' and type(received_value) is int: - success = received_value >= -140737488355327 and received_value <= 140737488355327 - elif constraint_value == 'nullable_int56s' and type(received_value) is int: - success = received_value >= -36028797018963967 and received_value <= 36028797018963967 - elif constraint_value == 'nullable_int64s' and type(received_value) is int: - success = received_value >= -9223372036854775807 and received_value <= 9223372036854775807 - else: - warning = True - message = f'{constraint} == {constraint_value}' - else: - message = f'Unknown constraint type: {constraint}' + # TODO eventually move getting contraints into __update_placeholder + # TODO We need to provide config to get_constraints and perform substitutions. + # if type(constraint_value) is str and constraint_value in config: + # constraint_value = config[constraint_value] + constraints = get_constraints(self.response['constraints']) - if success: - result.success(PostProcessCheckType.CONSTRAINT_VALIDATION, f'The test constraint "{message}" succeeds.') - elif warning: - result.warning(PostProcessCheckType.CONSTRAINT_VALIDATION, f'The test constraints "{message}" is ignored.') - else: - result.error(PostProcessCheckType.CONSTRAINT_VALIDATION, f'The test constraint "{message}" failed.') + received_value = response['value'] + if all([constraint.is_met(received_value) for constraint in constraints]): + result.success(PostProcessCheckType.CONSTRAINT_VALIDATION, f'Constraints check passed') + else: + # TODO would be helpful to be more verbose here + result.error(PostProcessCheckType.CONSTRAINT_VALIDATION, f'Constraints check failed') def __maybe_save_as(self, response, config, result): if not self.response or 'values' not in self.response: diff --git a/scripts/tests/yamltests/constraints.py b/scripts/tests/yamltests/constraints.py new file mode 100644 index 00000000000000..befb0b6b366c75 --- /dev/null +++ b/scripts/tests/yamltests/constraints.py @@ -0,0 +1,357 @@ +# +# Copyright (c) 2022 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from abc import ABC, abstractmethod +import string + + +class ConstraintParseError(Exception): + def __init__(self, message): + super().__init__(message) + + +class ConstraintValidationError(Exception): + def __init__(self, message): + super().__init__(message) + + +class BaseConstraint(ABC): + '''Constrain Interface''' + + def __init__(self, types: list, is_null_allowed: bool = False): + '''An empty type list provided that indicates any type is accepted''' + self._types = types + self._is_null_allowed = is_null_allowed + + def is_met(self, value): + response_type = type(value) + if self._types and response_type not in self._types: + return False + + if value is None: + return self._is_null_allowed + + return self.check_response(value) + + @abstractmethod + def check_response(self, value) -> bool: + pass + + +class _ConstraintHasValue(BaseConstraint): + def __init__(self, has_value): + super().__init__(types=[]) + self._has_value = has_value + + def check_response(self, value) -> bool: + raise ConstraintValidationError('HasValue constraint currently not implemented') + + +class _ConstraintType(BaseConstraint): + def __init__(self, type): + super().__init__(types=[], is_null_allowed=True) + self._type = type + + def check_response(self, value) -> bool: + success = False + if self._type == 'boolean' and type(value) is bool: + success = True + elif self._type == 'list' and type(value) is list: + success = True + elif self._type == 'char_string' and type(value) is str: + success = True + elif self._type == 'octet_string' and type(value) is bytes: + success = True + elif self._type == 'vendor_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFF + elif self._type == 'device_type_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'cluster_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'attribute_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'field_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'command_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'event_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'action_id' and type(value) is int: + success = value >= 0 and value <= 0xFF + elif self._type == 'transaction_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'node_id' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFFFFFF + elif self._type == 'bitmap8' and type(value) is int: + success = value >= 0 and value <= 0xFF + elif self._type == 'bitmap16' and type(value) is int: + success = value >= 0 and value <= 0xFFFF + elif self._type == 'bitmap32' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'bitmap64' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFFFFFF + elif self._type == 'enum8' and type(value) is int: + success = value >= 0 and value <= 0xFF + elif self._type == 'enum16' and type(value) is int: + success = value >= 0 and value <= 0xFFFF + elif self._type == 'Percent' and type(value) is int: + success = value >= 0 and value <= 0xFF + elif self._type == 'Percent100ths' and type(value) is int: + success = value >= 0 and value <= 0xFFFF + elif self._type == 'epoch_us' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFFFFFF + elif self._type == 'epoch_s' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'utc' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'date' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'tod' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'int8u' and type(value) is int: + success = value >= 0 and value <= 0xFF + elif self._type == 'int16u' and type(value) is int: + success = value >= 0 and value <= 0xFFFF + elif self._type == 'int24u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFF + elif self._type == 'int32u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFF + elif self._type == 'int40u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFF + elif self._type == 'int48u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFF + elif self._type == 'int56u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFFFF + elif self._type == 'int64u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFFFFFF + elif self._type == 'nullable_int8u' and type(value) is int: + success = value >= 0 and value <= 0xFE + elif self._type == 'nullable_int16u' and type(value) is int: + success = value >= 0 and value <= 0xFFFE + elif self._type == 'nullable_int24u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFE + elif self._type == 'nullable_int32u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFE + elif self._type == 'nullable_int40u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFE + elif self._type == 'nullable_int48u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFE + elif self._type == 'nullable_int56u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFFFE + elif self._type == 'nullable_int64u' and type(value) is int: + success = value >= 0 and value <= 0xFFFFFFFFFFFFFFFE + elif self._type == 'int8s' and type(value) is int: + success = value >= -128 and value <= 127 + elif self._type == 'int16s' and type(value) is int: + success = value >= -32768 and value <= 32767 + elif self._type == 'int24s' and type(value) is int: + success = value >= -8388608 and value <= 8388607 + elif self._type == 'int32s' and type(value) is int: + success = value >= -2147483648 and value <= 2147483647 + elif self._type == 'int40s' and type(value) is int: + success = value >= -549755813888 and value <= 549755813887 + elif self._type == 'int48s' and type(value) is int: + success = value >= -140737488355328 and value <= 140737488355327 + elif self._type == 'int56s' and type(value) is int: + success = value >= -36028797018963968 and value <= 36028797018963967 + elif self._type == 'int64s' and type(value) is int: + success = value >= -9223372036854775808 and value <= 9223372036854775807 + elif self._type == 'nullable_int8s' and type(value) is int: + success = value >= -127 and value <= 127 + elif self._type == 'nullable_int16s' and type(value) is int: + success = value >= -32767 and value <= 32767 + elif self._type == 'nullable_int24s' and type(value) is int: + success = value >= -8388607 and value <= 8388607 + elif self._type == 'nullable_int32s' and type(value) is int: + success = value >= -2147483647 and value <= 2147483647 + elif self._type == 'nullable_int40s' and type(value) is int: + success = value >= -549755813887 and value <= 549755813887 + elif self._type == 'nullable_int48s' and type(value) is int: + success = value >= -140737488355327 and value <= 140737488355327 + elif self._type == 'nullable_int56s' and type(value) is int: + success = value >= -36028797018963967 and value <= 36028797018963967 + elif self._type == 'nullable_int64s' and type(value) is int: + success = value >= -9223372036854775807 and value <= 9223372036854775807 + return success + + +class _ConstraintMinLength(BaseConstraint): + def __init__(self, min_length): + super().__init__(types=[str, bytes, list]) + self._min_length = min_length + + def check_response(self, value) -> bool: + return len(value) >= self._min_length + + +class _ConstraintMaxLength(BaseConstraint): + def __init__(self, max_length): + super().__init__(types=[str, bytes, list]) + self._max_length = max_length + + def check_response(self, value) -> bool: + return len(value) <= self._max_length + + +class _ConstraintIsHexString(BaseConstraint): + def __init__(self, is_hex_string: bool): + super().__init__(types=[str]) + self._is_hex_string = is_hex_string + + def check_response(self, value) -> bool: + return all(c in string.hexdigits for c in value) == self._is_hex_string + + +class _ConstraintStartsWith(BaseConstraint): + def __init__(self, starts_with): + super().__init__(types=[str]) + self._starts_with = starts_with + + def check_response(self, value) -> bool: + return value.startswith(self._starts_with) + + +class _ConstraintEndsWith(BaseConstraint): + def __init__(self, ends_with): + super().__init__(types=[str]) + self._ends_with = ends_with + + def check_response(self, value) -> bool: + return value.endswith(self._ends_with) + + +class _ConstraintIsUpperCase(BaseConstraint): + def __init__(self, is_upper_case): + super().__init__(types=[str]) + self._is_upper_case = is_upper_case + + def check_response(self, value) -> bool: + return value.isupper() == self._is_upper_case + + +class _ConstraintIsLowerCase(BaseConstraint): + def __init__(self, is_lower_case): + super().__init__(types=[str]) + self._is_lower_case = is_lower_case + + def check_response(self, value) -> bool: + return value.islower() == self._is_lower_case + + +class _ConstraintMinValue(BaseConstraint): + def __init__(self, min_value): + super().__init__(types=[int, float], is_null_allowed=True) + self._min_value = min_value + + def check_response(self, value) -> bool: + return value >= self._min_value + + +class _ConstraintMaxValue(BaseConstraint): + def __init__(self, max_value): + super().__init__(types=[int, float], is_null_allowed=True) + self._max_value = max_value + + def check_response(self, value) -> bool: + return value <= self._max_value + + +class _ConstraintContains(BaseConstraint): + def __init__(self, contains): + super().__init__(types=[list]) + self._contains = contains + + def check_response(self, value) -> bool: + return set(self._contains).issubset(value) + + +class _ConstraintExcludes(BaseConstraint): + def __init__(self, excludes): + super().__init__(types=[list]) + self._excludes = excludes + + def check_response(self, value) -> bool: + return set(self._excludes).isdisjoint(value) + + +class _ConstraintHasMaskSet(BaseConstraint): + def __init__(self, has_masks_set): + super().__init__(types=[int]) + self._has_masks_set = has_masks_set + + def check_response(self, value) -> bool: + return all([(value & mask) == mask for mask in self._has_masks_set]) + + +class _ConstraintHasMaskClear(BaseConstraint): + def __init__(self, has_masks_clear): + super().__init__(types=[int]) + self._has_masks_clear = has_masks_clear + + def check_response(self, value) -> bool: + return all([(value & mask) == 0 for mask in self._has_masks_clear]) + + +class _ConstraintNotValue(BaseConstraint): + def __init__(self, not_value): + super().__init__(types=[], is_null_allowed=True) + self._not_value = not_value + + def check_response(self, value) -> bool: + return value != self._not_value + + +def get_constraints(constraints: dict) -> list[BaseConstraint]: + _constraints = [] + + for constraint, constraint_value in constraints.items(): + if 'hasValue' == constraint: + _constraints.append(_ConstraintHasValue(constraint_value)) + elif 'type' == constraint: + _constraints.append(_ConstraintType(constraint_value)) + elif 'minLength' == constraint: + _constraints.append(_ConstraintMinLength(constraint_value)) + elif 'maxLength' == constraint: + _constraints.append(_ConstraintMaxLength(constraint_value)) + elif 'isHexString' == constraint: + _constraints.append(_ConstraintIsHexString(constraint_value)) + elif 'startsWith' == constraint: + _constraints.append(_ConstraintStartsWith(constraint_value)) + elif 'endsWith' == constraint: + _constraints.append(_ConstraintEndsWith(constraint_value)) + elif 'isUpperCase' == constraint: + _constraints.append(_ConstraintIsUpperCase(constraint_value)) + elif 'isLowerCase' == constraint: + _constraints.append(_ConstraintIsLowerCase(constraint_value)) + elif 'minValue' == constraint: + _constraints.append(_ConstraintMinValue(constraint_value)) + elif 'maxValue' == constraint: + _constraints.append(_ConstraintMaxValue(constraint_value)) + elif 'contains' == constraint: + _constraints.append(_ConstraintContains(constraint_value)) + elif 'excludes' == constraint: + _constraints.append(_ConstraintExcludes(constraint_value)) + elif 'hasMasksSet' == constraint: + _constraints.append(_ConstraintHasMaskSet(constraint_value)) + elif 'hasMasksClear' == constraint: + _constraints.append(_ConstraintHasMaskClear(constraint_value)) + elif 'notValue' == constraint: + _constraints.append(_ConstraintNotValue(constraint_value)) + else: + raise ConstraintParseError(f'Unknown constraint type:{constraint}') + + return _constraints From ce47d84ff87fb31ab6586990ad990fbaa849a7c9 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Tue, 6 Dec 2022 23:36:22 +0100 Subject: [PATCH 06/23] [Yaml Parser] Add scripts/tests/yamltests/SpecDefinitions.py with some unit tests --- scripts/tests/yamltests/SpecDefinitions.py | 199 ++++++++++++ .../tests/yamltests/test_spec_definitions.py | 291 ++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 scripts/tests/yamltests/SpecDefinitions.py create mode 100644 scripts/tests/yamltests/test_spec_definitions.py diff --git a/scripts/tests/yamltests/SpecDefinitions.py b/scripts/tests/yamltests/SpecDefinitions.py new file mode 100644 index 00000000000000..0bd70de5f2a769 --- /dev/null +++ b/scripts/tests/yamltests/SpecDefinitions.py @@ -0,0 +1,199 @@ +# +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List +import enum + +try: + from idl.zapxml import ParseSource, ParseXmls + from idl.matter_idl_types import * +except: + import os + import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) + + from idl.zapxml import ParseSource, ParseXmls + from idl.matter_idl_types import * + + +class _ItemType(enum.Enum): + Cluster = 0 + Request = 1 + Response = 2 + Attribute = 3 + Event = 4 + Bitmap = 5 + Enum = 6 + Struct = 7 + + +class SpecDefinitions: + __clusters_by_id: dict[int, Cluster] = {} + __commands_by_id: dict[int, dict[int, Command]] = {} + __responses_by_id: dict[int, dict[int, Struct]] = {} + __attributes_by_id: dict[int, dict[int, Attribute]] = {} + __events_by_id: dict[int, dict[int, Event]] = {} + + __clusters_by_name: dict[str, int] = {} + __commands_by_name: dict[str, int] = {} + __responses_by_name: dict[str, int] = {} + __attributes_by_name: dict[str, int] = {} + __events_by_name: dict[str, int] = {} + + __bitmaps_by_name: dict[str, dict[str, Bitmap]] = {} + __enums_by_name: dict[str, dict[str, Enum]] = {} + __structs_by_name: dict[str, dict[str, Struct]] = {} + + def __init__(self, sources: List[ParseSource]): + idl = ParseXmls(sources) + + for cluster in idl.clusters: + code: int = cluster.code + name: str = cluster.name + self.__clusters_by_id[code] = cluster + self.__commands_by_id[code] = {c.code: c for c in cluster.commands} + self.__responses_by_id[code] = {} + self.__attributes_by_id[code] = {a.definition.code: a for a in cluster.attributes} + self.__events_by_id[code] = {e.code: e for e in cluster.events} + + self.__clusters_by_name[name] = cluster.code + self.__commands_by_name[name] = {c.name.lower(): c.code for c in cluster.commands} + self.__responses_by_name[name] = {} + self.__attributes_by_name[name] = {a.definition.name.lower(): a.definition.code for a in cluster.attributes} + self.__events_by_name[name] = {e.name.lower(): e.code for e in cluster.events} + + self.__bitmaps_by_name[name] = {b.name.lower(): b for b in cluster.bitmaps} + self.__enums_by_name[name] = {e.name.lower(): e for e in cluster.enums} + self.__structs_by_name[name] = {s.name.lower(): s for s in cluster.structs} + + for struct in cluster.structs: + if struct.tag == StructTag.RESPONSE: + self.__responses_by_id[code][struct.code] = struct + self.__responses_by_name[name][struct.name.lower()] = struct.code + + def get_cluster_name(self, cluster_id: int) -> str: + cluster = self.__clusters_by_id.get(cluster_id) + return cluster.name if cluster else None + + def get_command_name(self, cluster_id: int, command_id: int) -> str: + command = self.__get_by_id(cluster_id, command_id, _ItemType.Request) + return command.name if command else None + + def get_response_name(self, cluster_id: int, response_id: int) -> str: + response = self.__get_by_id(cluster_id, response_id, _ItemType.Response) + return response.name if response else None + + def get_attribute_name(self, cluster_id: int, attribute_id: int) -> str: + attribute = self.__get_by_id(cluster_id, attribute_id, _ItemType.Attribute) + return attribute.definition.name if attribute else None + + def get_event_name(self, cluster_id: int, event_id: int) -> str: + event = self.__get_by_id(cluster_id, event_id, _ItemType.Event) + return event.name if event else None + + def get_command_by_name(self, cluster_name: str, command_name: str) -> Command: + return self.__get_by_name(cluster_name, command_name, _ItemType.Request) + + def get_response_by_name(self, cluster_name: str, response_name: str) -> Struct: + return self.__get_by_name(cluster_name, response_name, _ItemType.Response) + + def get_attribute_by_name(self, cluster_name: str, attribute_name: str) -> Attribute: + return self.__get_by_name(cluster_name, attribute_name, _ItemType.Attribute) + + def get_event_by_name(self, cluster_name: str, event_name: str) -> Event: + return self.__get_by_name(cluster_name, event_name, _ItemType.Event) + + def get_bitmap_by_name(self, cluster_name: str, bitmap_name: str) -> Bitmap: + return self.__get_by_name(cluster_name, bitmap_name, _ItemType.Bitmap) + + def get_enum_by_name(self, cluster_name: str, enum_name: str) -> Bitmap: + return self.__get_by_name(cluster_name, enum_name, _ItemType.Enum) + + def get_struct_by_name(self, cluster_name: str, struct_name: str) -> Struct: + return self.__get_by_name(cluster_name, struct_name, _ItemType.Struct) + + def get_type_by_name(self, cluster_name: str, target_name: str): + bitmap = self.get_bitmap_by_name(cluster_name, target_name) + if bitmap: + return bitmap + + enum = self.get_enum_by_name(cluster_name, target_name) + if enum: + return enum + + struct = self.get_struct_by_name(cluster_name, target_name) + if struct: + return struct + + return None + + def is_fabric_scoped(self, target) -> bool: + if hasattr(target, 'qualities'): + return bool(target.qualities & StructQuality.FABRIC_SCOPED) + return False + + def __get_by_name(self, cluster_name: str, target_name: str, target_type: _ItemType): + if not cluster_name or not target_name: + return None + + # The idl parser remove spaces + cluster_name = cluster_name.replace(' ', '') + # Many YAML tests formats the name using camelCase despites that the spec mandates + # CamelCase. To be compatible with the current tests, everything is converted to lower case. + target_name = target_name.lower() + + cluster_id = self.__clusters_by_name.get(cluster_name) + if cluster_id is None: + return None + + target = None + + if target_type == _ItemType.Request: + target_id = self.__commands_by_name.get(cluster_name).get(target_name) + target = self.__get_by_id(cluster_id, target_id, target_type) + elif target_type == _ItemType.Response: + target_id = self.__responses_by_name.get(cluster_name).get(target_name) + target = self.__get_by_id(cluster_id, target_id, target_type) + elif target_type == _ItemType.Event: + target_id = self.__events_by_name.get(cluster_name).get(target_name) + target = self.__get_by_id(cluster_id, target_id, target_type) + elif target_type == _ItemType.Attribute: + target_id = self.__attributes_by_name.get(cluster_name).get(target_name) + target = self.__get_by_id(cluster_id, target_id, target_type) + elif target_type == _ItemType.Bitmap: + target = self.__bitmaps_by_name.get(cluster_name).get(target_name) + elif target_type == _ItemType.Enum: + target = self.__enums_by_name.get(cluster_name).get(target_name) + elif target_type == _ItemType.Struct: + target = self.__structs_by_name.get(cluster_name).get(target_name) + + return target + + def __get_by_id(self, cluster_id: int, target_id: int, target_type: str): + targets = None + + if target_type == _ItemType.Request: + targets = self.__commands_by_id.get(cluster_id) + elif target_type == _ItemType.Response: + targets = self.__responses_by_id.get(cluster_id) + elif target_type == _ItemType.Event: + targets = self.__events_by_id.get(cluster_id) + elif target_type == _ItemType.Attribute: + targets = self.__attributes_by_id.get(cluster_id) + + if targets is None: + return None + + return targets.get(target_id) diff --git a/scripts/tests/yamltests/test_spec_definitions.py b/scripts/tests/yamltests/test_spec_definitions.py new file mode 100644 index 00000000000000..fbf480c6beb083 --- /dev/null +++ b/scripts/tests/yamltests/test_spec_definitions.py @@ -0,0 +1,291 @@ +#!/usr/bin/env -S python3 -B +# +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from SpecDefinitions import * + +import unittest +import io + +source_cluster = ''' + + + Test + 0x1234 + + +''' + +source_command = ''' + + + Test + 0x1234 + + + + + +''' + +source_response = ''' + + + Test + 0x1234 + + + + + + + +''' + +source_attribute = ''' + + + TestGlobalAttribute + + + + Test + 0x1234 + + + TestAttribute + + + +''' + +source_event = ''' + + + Test + 0x1234 + + + + + +''' + +source_bitmap = ''' + + + + + + + + + + + + + Test + 0x1234 + + + + TestWrong + 0x4321 + + +''' + +source_enum = ''' + + + + + + + + + + + + + Test + 0x1234 + + + + TestWrong + 0x4321 + + +''' + +source_struct = ''' + + + + + + + + + + + + + + + + + + Test + 0x1234 + + + + TestWrong + 0x4321 + + +''' + + +class TestSpecDefinitions(unittest.TestCase): + def test_cluster_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_cluster), name='source_cluster')]) + self.assertIsNone(definitions.get_cluster_name(0x4321)) + self.assertEqual(definitions.get_cluster_name(0x1234), 'Test') + + def test_command_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_command), name='source_command')]) + self.assertIsNone(definitions.get_command_name(0x4321, 0x0)) + self.assertIsNone(definitions.get_command_name(0x1234, 0x1)) + self.assertEqual(definitions.get_command_name(0x1234, 0x0), 'TestCommand') + + def test_response_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_response), name='source_response')]) + self.assertIsNone(definitions.get_response_name(0x4321, 0x0)) + self.assertIsNone(definitions.get_response_name(0x1234, 0x1)) + self.assertEqual(definitions.get_response_name(0x1234, 0x0), 'TestCommandResponse') + + def test_attribute_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_attribute), name='source_attribute')]) + self.assertIsNone(definitions.get_attribute_name(0x4321, 0x0)) + self.assertIsNone(definitions.get_attribute_name(0x4321, 0xFFFD)) + self.assertIsNone(definitions.get_attribute_name(0x1234, 0x1)) + self.assertEqual(definitions.get_attribute_name(0x1234, 0x0), 'TestAttribute') + self.assertEqual(definitions.get_attribute_name(0x1234, 0xFFFD), 'TestGlobalAttribute') + + def test_event_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_event), name='source_event')]) + self.assertIsNone(definitions.get_event_name(0x4321, 0x0)) + self.assertIsNone(definitions.get_event_name(0x1234, 0x1)) + self.assertEqual(definitions.get_event_name(0x1234, 0x0), 'TestEvent') + + def test_get_command_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_command), name='source_command')]) + self.assertIsNone(definitions.get_command_by_name('WrongName', 'TestCommand')) + self.assertIsNone(definitions.get_command_by_name('Test', 'TestWrongCommand')) + self.assertIsNone(definitions.get_response_by_name('Test', 'TestCommand')) + self.assertIsInstance(definitions.get_command_by_name('Test', 'TestCommand'), Command) + self.assertIsNone(definitions.get_command_by_name('test', 'TestCommand')) + self.assertIsInstance(definitions.get_command_by_name('Test', 'testcommand'), Command) + + def test_get_response_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_response), name='source_response')]) + self.assertIsNone(definitions.get_response_by_name('WrongName', 'TestCommandResponse')) + self.assertIsNone(definitions.get_response_by_name('Test', 'TestWrongCommandResponse')) + self.assertIsNone(definitions.get_command_by_name('Test', 'TestCommandResponse')) + self.assertIsInstance(definitions.get_response_by_name('Test', 'TestCommandResponse'), Struct) + self.assertIsNone(definitions.get_response_by_name('test', 'TestCommandResponse')) + self.assertIsInstance(definitions.get_response_by_name('Test', 'testcommandresponse'), Struct) + + def test_get_attribute_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_attribute), name='source_attribute')]) + self.assertIsNone(definitions.get_attribute_by_name('WrongName', 'TestAttribute')) + self.assertIsNone(definitions.get_attribute_by_name('WrongName', 'TestGlobalAttribute')) + self.assertIsNone(definitions.get_attribute_by_name('Test', 'TestWrongAttribute')) + self.assertIsInstance(definitions.get_attribute_by_name('Test', 'TestAttribute'), Attribute) + self.assertIsInstance(definitions.get_attribute_by_name('Test', 'TestGlobalAttribute'), Attribute) + self.assertIsNone(definitions.get_attribute_by_name('test', 'TestAttribute')) + self.assertIsNone(definitions.get_attribute_by_name('test', 'TestGlobalAttribute')) + self.assertIsInstance(definitions.get_attribute_by_name('Test', 'testattribute'), Attribute) + self.assertIsInstance(definitions.get_attribute_by_name('Test', 'testglobalattribute'), Attribute) + + def test_get_event_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_event), name='source_event')]) + self.assertIsNone(definitions.get_event_by_name('WrongName', 'TestEvent')) + self.assertIsNone(definitions.get_event_by_name('Test', 'TestWrongEvent')) + self.assertIsInstance(definitions.get_event_by_name('Test', 'TestEvent'), Event) + self.assertIsNone(definitions.get_event_by_name('test', 'TestEvent')) + self.assertIsInstance(definitions.get_event_by_name('Test', 'testevent'), Event) + + def test_get_bitmap_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_bitmap), name='source_bitmap')]) + self.assertIsNone(definitions.get_bitmap_by_name('WrongName', 'TestBitmap')) + self.assertIsNone(definitions.get_bitmap_by_name('Test', 'TestWrongBitmap')) + self.assertIsInstance(definitions.get_bitmap_by_name('Test', 'TestBitmap'), Bitmap) + self.assertIsNone(definitions.get_bitmap_by_name('test', 'TestBitmap')) + self.assertIsInstance(definitions.get_bitmap_by_name('Test', 'testbitmap'), Bitmap) + + def test_get_enum_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_enum), name='source_enum')]) + self.assertIsNone(definitions.get_enum_by_name('WrongName', 'TestEnum')) + self.assertIsNone(definitions.get_enum_by_name('Test', 'TestWrongEnum')) + self.assertIsInstance(definitions.get_enum_by_name('Test', 'TestEnum'), Enum) + self.assertIsNone(definitions.get_enum_by_name('test', 'TestEnum')) + self.assertIsInstance(definitions.get_enum_by_name('Test', 'testenum'), Enum) + + def test_get_struct_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_struct), name='source_struct')]) + self.assertIsNone(definitions.get_struct_by_name('WrongName', 'TestStruct')) + self.assertIsNone(definitions.get_struct_by_name('Test', 'TestWrongStruct')) + self.assertIsInstance(definitions.get_struct_by_name('Test', 'TestStruct'), Struct) + self.assertIsNone(definitions.get_struct_by_name('test', 'TestStruct')) + self.assertIsInstance(definitions.get_struct_by_name('Test', 'teststruct'), Struct) + + def test_get_type_by_name(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_command), name='source_command')]) + self.assertIsNone(definitions.get_type_by_name('Test', 'TestCommand')) + + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_response), name='source_response')]) + self.assertIsInstance(definitions.get_type_by_name('Test', 'TestCommandResponse'), Struct) + + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_attribute), name='source_attribute')]) + self.assertIsNone(definitions.get_type_by_name('Test', 'TestAttribute')) + + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_event), name='source_event')]) + self.assertIsNone(definitions.get_type_by_name('Test', 'TestEvent')) + + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_bitmap), name='source_bitmap')]) + self.assertIsInstance(definitions.get_type_by_name('Test', 'TestBitmap'), Bitmap) + + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_enum), name='source_enum')]) + self.assertIsInstance(definitions.get_type_by_name('Test', 'TestEnum'), Enum) + + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_struct), name='source_struct')]) + self.assertIsInstance(definitions.get_type_by_name('Test', 'TestStruct'), Struct) + + def test_is_fabric_scoped(self): + definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_struct), name='source_struct')]) + + struct = definitions.get_struct_by_name('Test', 'TestStruct') + self.assertFalse(definitions.is_fabric_scoped(struct)) + + struct = definitions.get_struct_by_name('Test', 'TestStructFabricScoped') + self.assertTrue(definitions.is_fabric_scoped(struct)) + + +if __name__ == '__main__': + unittest.main() From b06cbd2adb7d19b10791883ed0afcdc9a3286fe4 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Tue, 6 Dec 2022 23:40:56 +0100 Subject: [PATCH 07/23] [Yaml Parser] Use SpecDefinitions API instead of ClusterDefinitions --- scripts/tests/test_yaml_parser.py | 24 +- .../tests/yamltests/ClustersDefinitions.py | 261 ------------------ scripts/tests/yamltests/YamlParser.py | 31 ++- 3 files changed, 45 insertions(+), 271 deletions(-) delete mode 100644 scripts/tests/yamltests/ClustersDefinitions.py diff --git a/scripts/tests/test_yaml_parser.py b/scripts/tests/test_yaml_parser.py index f864c6dadf973a..e9154b1fca6ec3 100644 --- a/scripts/tests/test_yaml_parser.py +++ b/scripts/tests/test_yaml_parser.py @@ -19,11 +19,13 @@ # to a more appropriate spot. For now, having this file to do some quick checks # is arguably better then no checks at all. +import glob import os import unittest +import functools from pathlib import Path -from yamltests.ClustersDefinitions import ClustersDefinitions +from yamltests.SpecDefinitions import * from yamltests.YamlParser import YamlParser _DEFAULT_MATTER_ROOT = os.path.abspath( @@ -35,6 +37,19 @@ os.path.join(_DEFAULT_MATTER_ROOT, 'src/app/zap-templates/zcl/data-model')) +def sort_with_global_attribute_first(a, b): + if a.endswith('global-attributes.xml'): + return -1 + elif b.endswith('global-attributes.xml'): + return 1 + elif a > b: + return 1 + elif a == b: + return 0 + elif a < b: + return -1 + + class TestYamlParser(unittest.TestCase): def setUp(self): # TODO we should not be reliant on an external YAML file. Possible that we should have @@ -60,9 +75,12 @@ def setUp(self): # TODO Again we should not be reliant on extneral XML files. But some test (even brittal) # are better than no tests. - clusters_definitions = ClustersDefinitions(_CLUSTER_DEFINITION_DIRECTORY) + filenames = glob.glob(_CLUSTER_DEFINITION_DIRECTORY + '/*/*.xml', recursive=False) + filenames.sort(key=functools.cmp_to_key(sort_with_global_attribute_first)) + sources = [ParseSource(source=name) for name in filenames] + specifications = SpecDefinitions(sources) - self._yaml_parser = YamlParser(path_to_test, pics_file, clusters_definitions) + self._yaml_parser = YamlParser(path_to_test, pics_file, specifications) def test_able_to_iterate_over_all_tests(self): # self._yaml_parser.tests implements `__next__`, which does value substitution. We are diff --git a/scripts/tests/yamltests/ClustersDefinitions.py b/scripts/tests/yamltests/ClustersDefinitions.py deleted file mode 100644 index db05922abd2117..00000000000000 --- a/scripts/tests/yamltests/ClustersDefinitions.py +++ /dev/null @@ -1,261 +0,0 @@ -# -# Copyright (c) 2022 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import xml.etree.ElementTree as ET -import glob -import time - - -# TODO(use scripts/idl/xml_parser.py for zap xml parsing) - -class ClustersDefinitions: - def __init__(self, clusters_dir): - self.__clusterIdToName = {} - self.__commandIdToName = {} - self.__attributeIdToName = {} - self.__globalAttributeIdToName = {} - self.__eventIdToName = {} - - self.__commands = {} - self.__responses = {} - self.__attributes = {} - self.__events = {} - self.__structs = {} - self.__enums = {} - self.__bitmaps = {} - - clusters_files = glob.glob(clusters_dir + '**/*.xml', recursive=True) - for cluster_file in clusters_files: - tree = ET.parse(cluster_file) - - root = tree.getroot() - - structs = root.findall('struct') - for struct in structs: - self.__structs[struct.get('name').lower()] = struct - - enums = root.findall('enum') - for enum in enums: - self.__enums[enum.get('name').lower()] = enum - - bitmaps = root.findall('bitmap') - for bitmap in bitmaps: - self.__bitmaps[bitmap.get('name').lower()] = bitmap - - global_elements = root.find('global') - if global_elements: - global_attributes = global_elements.findall('attribute') - - for attribute in global_attributes: - attribute_side = attribute.get('side') - attribute_code = attribute.get('code') - - description = attribute.find('description') - attribute_name = description.text if description is not None else attribute.text - - if attribute_code.startswith('0x') or attribute_code.startswith('0X'): - attribute_code = int(attribute_code, base=16) - self.__globalAttributeIdToName[attribute_code] = attribute_name - - if attribute_side == 'server': - self.__attributes[attribute_name.lower()] = attribute - - cluster = root.find('cluster') - if not cluster: - continue - - cluster_code = cluster.find('code').text - cluster_name = cluster.find('name').text - - if cluster_code.startswith('0x') or cluster_code.startswith('0X'): - cluster_code = int(cluster_code, base=16) - - self.__clusterIdToName[cluster_code] = cluster_name - self.__commandIdToName[cluster_code] = {} - self.__attributeIdToName[cluster_code] = {} - self.__eventIdToName[cluster_code] = {} - - commands = cluster.findall('command') - for command in commands: - command_source = command.get('source') - command_code = command.get('code') - command_name = command.get('name') - - base = 16 if command_code.startswith( - '0x') or command_code.startswith('0X') else 10 - command_code = int(command_code, base=base) - self.__commandIdToName[cluster_code][command_code] = command_name - - if command_source == 'client': - self.__commands[command_name.lower()] = command - elif command_source == 'server': - # The name is not converted to lowercase here - self.__responses[command_name] = command - - attributes = cluster.findall('attribute') - for attribute in attributes: - attribute_side = attribute.get('side') - attribute_code = attribute.get('code') - - description = attribute.find('description') - attribute_name = description.text if description is not None else attribute.text - - base = 16 if attribute_code.startswith( - '0x') or attribute_code.startswith('0X') else 10 - attribute_code = int(attribute_code, base=base) - self.__attributeIdToName[cluster_code][attribute_code] = attribute_name - - if attribute_side == 'server': - self.__attributes[attribute_name.lower()] = attribute - - events = cluster.findall('event') - for event in events: - event_side = event.get('side') - event_code = event.get('code') - - description = event.find('description') - event_name = description.text if description is not None else event.text - - base = 16 if event_code.startswith( - '0x') or event_code.startswith('0X') else 10 - event_code = int(event_code, base=base) - self.__eventIdToName[cluster_code][event_code] = event_name - - if event_side == 'server': - self.__events[event_name.lower()] = event - - def get_cluster_name(self, cluster_id): - return self.__clusterIdToName[cluster_id] - - def get_command_name(self, cluster_id, command_id): - return self.__commandIdToName[cluster_id][command_id] - - def get_attribute_name(self, cluster_id, attribute_id): - if attribute_id in self.__globalAttributeIdToName: - return self.__globalAttributeIdToName[attribute_id] - return self.__attributeIdToName[cluster_id][attribute_id] - - def get_event_name(self, cluster_id, event_id): - return self.__eventIdToName[cluster_id][event_id] - - def get_response_mapping(self, command_name): - if not command_name in self.__responses: - return None - response = self.__responses[command_name] - - args = response.findall('arg') - - mapping = {} - for mapping_index, arg in enumerate(args): - mapping[str(mapping_index)] = {'name': arg.get( - 'name'), 'type': arg.get('type').lower()} - return mapping - - def get_attribute_mapping(self, attribute_name): - if not attribute_name.lower() in self.__attributes: - return None - attribute = self.__attributes[attribute_name.lower()] - - attribute_type = attribute.get('type') - if attribute_type.lower() == 'array': - attribute_type = attribute.get('entryType') - - if not self.get_type_mapping(attribute_type): - return None - - return {'name': attribute_name.lower(), 'type': attribute_type} - - def get_event_mapping(self, event_name): - return None - - def get_type_mapping(self, type_name): - struct = self.__structs.get(type_name.lower()) - if struct is None: - return None - - mapping = {} - - # FabricIndex is a special case where the framework requires it to be passed even if it is not part of the - # requested arguments per spec and not part of the XML definition. - if struct.get('isFabricScoped'): - mapping[str(254)] = {'name': 'FabricIndex', 'type': 'int64u'} - - mapping_index = 0 - - items = struct.findall('item') - for item in items: - if item.get('fieldId'): - mapping_index = int(item.get('fieldId')) - mapping[str(mapping_index)] = {'name': item.get( - 'name'), 'type': item.get('type').lower()} - mapping_index += 1 - - return mapping - - def get_attribute_definition(self, attribute_name): - attribute = self.__attributes.get(attribute_name.lower()) - if attribute is None: - return None - - attribute_type = attribute.get('type').lower() - if attribute_type == 'array': - attribute_type = attribute.get('entryType').lower() - return self.__to_base_type(attribute_type) - - def get_command_args_definition(self, command_name): - command = self.__commands.get(command_name.lower()) - if command is None: - return None - - return self.__args_to_dict(command.findall('arg')) - - def get_response_args_definition(self, command_name): - command = self.__commands.get(command_name.lower()) - if command is None: - return None - - response = self.__responses.get(command.get('response')) - if response is None: - return None - - return self.__args_to_dict(response.findall('arg')) - - def __args_to_dict(self, args): - rv = {} - for item in args: - arg_name = item.get('name') - arg_type = item.get('type').lower() - rv[arg_name] = self.__to_base_type(arg_type) - return rv - - def __to_base_type(self, item_type): - if item_type in self.__bitmaps: - bitmap = self.__bitmaps[item_type] - item_type = bitmap.get('type').lower() - elif item_type in self.__enums: - enum = self.__enums[item_type] - item_type = enum.get('type').lower() - elif item_type in self.__structs: - struct = self.__structs[item_type] - item_type = self.__struct_to_dict(struct) - return item_type - - def __struct_to_dict(self, struct): - type_entry = {} - for item in struct.findall('item'): - item_name = item.get('name') - item_type = item.get('type').lower() - type_entry[item_name] = self.__to_base_type(item_type) - return type_entry diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index 8db1557a7bc688..17fca44b389d0a 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -210,14 +210,19 @@ def __init__(self, test: dict, config: dict, definitions): _check_valid_keys(self.response, _TEST_RESPONSE_SECTION) if self.is_attribute: - attribute_definition = definitions.get_attribute_definition(self.attribute) - self.arguments = self.__update_with_definition(self.arguments, attribute_definition, config) - self.response = self.__update_with_definition(self.response, attribute_definition, config) + attribute = definitions.get_attribute_by_name(self.cluster, self.attribute) + if attribute: + attribute_definition = self.__as_mapping(definitions, self.cluster, attribute.definition.data_type.name) + self.arguments = self.__update_with_definition(self.arguments, attribute_definition, config) + self.response = self.__update_with_definition(self.response, attribute_definition, config) else: - command_definition = definitions.get_command_args_definition(self.command) - response_definition = definitions.get_response_args_definition(self.command) - self.arguments = self.__update_with_definition(self.arguments, command_definition, config) - self.response = self.__update_with_definition(self.response, response_definition, config) + command = definitions.get_command_by_name(self.cluster, self.command) + if command: + command_definition = self.__as_mapping(definitions, self.cluster, command.input_param) + response_definition = self.__as_mapping(definitions, self.cluster, command.output_param) + + self.arguments = self.__update_with_definition(self.arguments, command_definition, config) + self.response = self.__update_with_definition(self.response, response_definition, config) def post_process_response(self, response, config): result = PostProcessResponseResult() @@ -375,6 +380,18 @@ def __maybe_save_as(self, response, config, result): config[saveAs] = received_value result.success(PostProcessCheckType.SAVE_AS_VARIABLE, f'The test save the value "{received_value}" as {saveAs}.') + def __as_mapping(self, definitions, cluster_name, target_name): + element = definitions.get_type_by_name(cluster_name, target_name) + + if hasattr(element, 'base_type'): + target_name = element.base_type.lower() + elif hasattr(element, 'fields'): + target_name = {f.name: self.__as_mapping(definitions, cluster_name, f.data_type.name) for f in element.fields} + elif target_name: + target_name = target_name.lower() + + return target_name + def __update_with_definition(self, container, mapping_type, config): if not container or not mapping_type: return container From c680b7b667ba5418cf04d8e2f4fcf15f24c28226 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Tue, 6 Dec 2022 23:47:29 +0100 Subject: [PATCH 08/23] [Yaml Parser] Uses convert_yaml_octet_string_to_bytes for octet_string --- scripts/tests/yamltests/YamlFixes.py | 22 ++++++++++++++++++++++ scripts/tests/yamltests/YamlParser.py | 5 +---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/tests/yamltests/YamlFixes.py b/scripts/tests/yamltests/YamlFixes.py index 019d1450417d66..f8e3b54a0aaee8 100644 --- a/scripts/tests/yamltests/YamlFixes.py +++ b/scripts/tests/yamltests/YamlFixes.py @@ -14,6 +14,7 @@ # limitations under the License. import re +import binascii # Some of the YAML files contains some values that has been crafted to avoid some limitations # of the original JavaScript parser and the C++ generated bits. @@ -42,6 +43,27 @@ def try_apply_yaml_float_written_as_strings(value): value = float(value) return value + +def convert_yaml_octet_string_to_bytes(s: str) -> bytes: + # This method is a clone of the method in src/controller/python/chip/yaml/format_converter.py + # It needs to be only a single copy of this method. + """Convert YAML octet string body to bytes, handling any c-style hex escapes (e.g. \x5a) and hex: prefix""" + # Step 1: handle explicit "hex:" prefix + if s.startswith('hex:'): + return binascii.unhexlify(s[4:]) + + # Step 2: convert non-hex-prefixed to bytes + # TODO(#23669): This does not properly support utf8 octet strings. We mimic + # javascript codegen behavior. Behavior of javascript is: + # * Octet string character >= u+0200 errors out. + # * Any character greater than 0xFF has the upper bytes chopped off. + as_bytes = [ord(c) for c in s] + + if any([value > 0x200 for value in as_bytes]): + raise ValueError('Unsupported char in octet string %r' % as_bytes) + accumulated_hex = ''.join([f"{(v & 0xFF):02x}" for v in as_bytes]) + return binascii.unhexlify(accumulated_hex) + # The PyYAML implementation does not match float according to the JSON and fails on valid numbers. diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index 17fca44b389d0a..ca397711253777 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -446,10 +446,7 @@ def __update_value_with_definition(self, value, mapping_type, config): elif mapping_type == 'single' or mapping_type == 'double': value = YamlFixes.try_apply_yaml_float_written_as_strings(value) elif mapping_type == 'octet_string' or mapping_type == 'long_octet_string': - if value.startswith('hex:'): - value = bytes.fromhex(value[4:]) - else: - value = value.encode() + value = YamlFixes.convert_yaml_octet_string_to_bytes(value) elif mapping_type == 'boolean': value = bool(value) From 1115306db7c0d90a658e418c3e3ff37217ba846f Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Wed, 7 Dec 2022 14:46:06 +0100 Subject: [PATCH 09/23] [Yaml Parser / Constraints] Move the check for None before the type checking --- scripts/tests/yamltests/constraints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/tests/yamltests/constraints.py b/scripts/tests/yamltests/constraints.py index befb0b6b366c75..0da769fb321d83 100644 --- a/scripts/tests/yamltests/constraints.py +++ b/scripts/tests/yamltests/constraints.py @@ -38,13 +38,13 @@ def __init__(self, types: list, is_null_allowed: bool = False): self._is_null_allowed = is_null_allowed def is_met(self, value): + if value is None: + return self._is_null_allowed + response_type = type(value) if self._types and response_type not in self._types: return False - if value is None: - return self._is_null_allowed - return self.check_response(value) @abstractmethod From 1c5f1cb30b0b79a736da2db13fc842f078327ac6 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Tue, 6 Dec 2022 23:41:45 +0100 Subject: [PATCH 10/23] [Yaml Parser] Add _convert_single_value_to_values and make it sooner such that some code can be made simpler --- scripts/tests/yamltests/YamlParser.py | 295 ++++++++++++++------------ 1 file changed, 158 insertions(+), 137 deletions(-) diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index ca397711253777..5f5c131b272c52 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -57,6 +57,7 @@ 'value', 'values', 'error', + 'clusterError', 'constraints', 'type', 'hasMasksSet', @@ -68,6 +69,7 @@ 'readAttribute', 'writeAttribute', 'subscribeAttribute', + 'waitForReport', ] _EVENT_COMMANDS = [ @@ -224,17 +226,47 @@ def __init__(self, test: dict, config: dict, definitions): self.arguments = self.__update_with_definition(self.arguments, command_definition, config) self.response = self.__update_with_definition(self.response, response_definition, config) + # TODO Move this calls before the previous blocks that calls update_with_definition to make the + # underlying update_with_definition logic simpler to read. + self._convert_single_value_to_values(self.arguments) + self._convert_single_value_to_values(self.response) + + def _convert_single_value_to_values(self, container): + if container is None or 'values' in container: + return + + # Attribute tests pass a single value argument that does not carry a name but + # instead uses a generic 'value' keyword. Convert to keyword to be the single + # members of the 'values' array which is what is used for other tests. + value = {} + + known_keys_to_copy = ['value', 'constraints', 'saveAs'] + known_keys_to_allow = ['error', 'clusterError'] + + for key, item in list(container.items()): + if key in known_keys_to_copy: + value[key] = item + del container[key] + elif key in known_keys_to_allow: + # Nothing to do for those keys. + pass + else: + raise KeyError(f'Unknown key: {key}') + + container['values'] = [value] + def post_process_response(self, response, config): result = PostProcessResponseResult() - if (not self.__maybe_check_optional(response, result)): + if not self.__maybe_check_optional(response, result): return result self.__maybe_check_error(response, result) - self.__maybe_check_cluster_error(response, result) - self.__maybe_check_values(response, config, result) - self.__maybe_check_constraints(response, config, result) - self.__maybe_save_as(response, config, result) + if self.response: + self.__maybe_check_cluster_error(response, result) + self.__maybe_check_values(response, config, result) + self.__maybe_check_constraints(response, config, result) + self.__maybe_save_as(response, config, result) return result @@ -242,143 +274,141 @@ def __maybe_check_optional(self, response, result): if not self.optional or not 'error' in response: return True - error = response['error'] - if error == 'UNSUPPORTED_ATTRIBUTE' or error == 'UNSUPPORTED_COMMAND': + received_error = response.get('error') + if received_error == 'UNSUPPORTED_ATTRIBUTE' or received_error == 'UNSUPPORTED_COMMAND': # result.warning('Optional', f'The response contains the error: "{error}".') return False return True def __maybe_check_error(self, response, result): - if self.response and 'error' in self.response: - expectedError = self.response['error'] - if 'error' in response: - receivedError = response['error'] - if expectedError == receivedError: - result.success(PostProcessCheckType.IM_STATUS, - f'The test expects the "{expectedError}" error which occured successfully.') - else: - result.error(PostProcessCheckType.IM_STATUS, - f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') - else: - result.error(PostProcessCheckType.IM_STATUS, f'The test expects the "{expectedError}" error but no error occured.') - elif not self.response or not 'error' in self.response: - if 'error' in response: - receivedError = response['error'] - result.error(PostProcessCheckType.IM_STATUS, f'The test expects no error but the "{receivedError}" error occured.') - # Handle generic success/errors - elif response == 'failure': - receivedError = response - result.error(PostProcessCheckType.IM_STATUS, f'The test expects no error but the "{receivedError}" error occured.') - else: - result.success(PostProcessCheckType.IM_STATUS, f'The test expects no error an no error occured.') + check_type = PostProcessCheckType.IM_STATUS + error_success = 'The test expects the "{error}" error which occured successfully.' + error_success_no_error = 'The test expects no error and no error occurred.' + error_wrong_error = 'The test expects the "{error}" error but the "{value}" error occured.' + error_unexpected_error = 'The test expects no error but the "{value}" error occured.' + error_unexpected_success = 'The test expects the "{error}" error but no error occured.' + + expected_error = self.response.get('error') if self.response else None + + # Handle generic success/error + if type(response) is str and response == 'failure': + received_error = response + elif type(response) is str and response == 'success': + received_error = None + else: + received_error = response.get('error') + + if expected_error and received_error and expected_error == received_error: + result.success(check_type, error_success.format(error=expected_error)) + elif expected_error and received_error: + result.error(check_type, error_wrong_error.format(error=expected_error, value=received_error)) + elif expected_error and not received_error: + result.error(check_type, error_unexpected_success.format(error=expected_error)) + elif not expected_error and received_error: + result.error(check_type, error_unexpected_error.format(error=received_error)) + elif not expected_error and not received_error: + result.success(check_type, error_success_no_error) + else: + # This should not happens + raise AssertionError('This should not happens.') def __maybe_check_cluster_error(self, response, result): - if self.response and 'clusterError' in self.response: - expectedError = self.response['clusterError'] - if 'clusterError' in response: - receivedError = response['clusterError'] - if expectedError == receivedError: - result.success(PostProcessCheckType.CLUSTER_STATUS, - f'The test expects the "{expectedError}" error which occured successfully.') - else: - result.error(PostProcessCheckType.CLUSTER_STATUS, - f'The test expects the "{expectedError}" error but the "{receivedError}" error occured.') + check_type = PostProcessCheckType.IM_STATUS + error_success = 'The test expects the "{error}" error which occured successfully.' + error_unexpected_success = 'The test expects the "{error}" error but no error occured.' + error_wrong_error = 'The test expects the "{error}" error but the "{value}" error occured.' + + expected_error = self.response.get('clusterError') + received_error = response.get('clusterError') + + if expected_error: + if received_error and expected_error == received_error: + result.success(check_type, error_success.format(error=expected_error)) + elif received_error: + result.error(check_type, error_wrong_error.format(error=expected_error, value=received_error)) else: - result.error(PostProcessCheckType.CLUSTER_STATUS, - f'The test expects the "{expectedError}" error but no error occured.') + result.error(check_type, error_unexpected_success.format(error=expected_error)) + else: + # Nothing is logged here to not be redundant with the generic error checking code. + pass def __maybe_check_values(self, response, config, result): - if not self.response or not 'values' in self.response: - return + check_type = PostProcessCheckType.RESPONSE_VALIDATION + error_success = 'The test expectation "{name} == {value}" is true' + error_failure = 'The test expectation "{name} == {value}" is false' + error_name_does_not_exist = 'The test expects a value named "{name}" but it does not exists in the response."' - if not 'value' in response: - result.error(PostProcessCheckType.RESPONSE_VALIDATION, f'The test expects some values but none was received.') - return - - expected_entries = self.response['values'] - received_entry = response['value'] - - for expected_entry in expected_entries: - if type(expected_entry) is dict and type(received_entry) is dict: - if not 'value' in expected_entry: - continue + for value in self.response['values']: + if not 'value' in value: + continue - expected_name = expected_entry['name'] - expected_value = expected_entry['value'] - if not expected_name in received_entry: - if expected_value is None: - result.success(PostProcessCheckType.RESPONSE_VALIDATION, - f'The test expectation "{expected_name} == {expected_value}" is true') - else: - result.error( - PostProcessCheckType.RESPONSE_VALIDATION, f'The test expects a value named "{expected_name}" but it does not exists in the response."') - return - - received_value = received_entry[expected_name] - self.__check_value(expected_name, expected_value, received_value, result) - else: - if not 'value' in expected_entries[0]: + expected_name = 'value' + received_value = response.get('value') + if not self.is_attribute: + expected_name = value.get('name') + if not expected_name in received_value: + result.error(check_type, error_name_does_not_exist.format(name=expected_name)) continue - expected_value = expected_entry['value'] - received_value = received_entry - self.__check_value('value', expected_value, received_value, result) + received_value = received_value.get(expected_name) if received_value else None - def __check_value(self, name, expected_value, received_value, result): - # TODO Supports Array/List. See an exemple of failure in TestArmFailSafe.yaml - if expected_value == received_value: - result.success(PostProcessCheckType.RESPONSE_VALIDATION, f'The test expectation "{name} == {expected_value}" is true') - else: - result.error(PostProcessCheckType.RESPONSE_VALIDATION, f'The test expectation "{name} == {expected_value}" is false') + # TODO Supports Array/List. See an exemple of failure in TestArmFailSafe.yaml + expected_value = value.get('value') + if expected_value == received_value: + result.success(check_type, error_success.format(name=expected_name, value=received_value)) + else: + result.error(check_type, error_failure.format(name=expected_name, value=expected_value)) def __maybe_check_constraints(self, response, config, result): - if not self.response or not 'constraints' in self.response: - return + check_type = PostProcessCheckType.CONSTRAINT_VALIDATION + error_success = 'Constraints check passed' + error_failure = 'Constraints check failed' + error_name_does_not_exist = 'The test expects a value named "{name}" but it does not exists in the response."' - # TODO eventually move getting contraints into __update_placeholder - # TODO We need to provide config to get_constraints and perform substitutions. - # if type(constraint_value) is str and constraint_value in config: - # constraint_value = config[constraint_value] - constraints = get_constraints(self.response['constraints']) + for value in self.response['values']: + if not 'constraints' in value: + continue - received_value = response['value'] - if all([constraint.is_met(received_value) for constraint in constraints]): - result.success(PostProcessCheckType.CONSTRAINT_VALIDATION, f'Constraints check passed') - else: - # TODO would be helpful to be more verbose here - result.error(PostProcessCheckType.CONSTRAINT_VALIDATION, f'Constraints check failed') + expected_name = 'value' + received_value = response.get('value') + if not self.is_attribute: + expected_name = value.get('name') + if not expected_name in received_value: + result.error(check_type, error_name_does_not_exist.format(name=expected_name)) + continue - def __maybe_save_as(self, response, config, result): - if not self.response or 'values' not in self.response: - return + received_value = received_value.get(expected_name) if received_value else None - if not 'value' in response: - result.error(PostProcessCheckType.SAVE_AS_VARIABLE, f'The test expects some values but none was received.') - return + constraints = get_constraints(value['constraints']) + if all([constraint.is_met(received_value) for constraint in constraints]): + result.success(check_type, error_success) + else: + # TODO would be helpful to be more verbose here + result.error(check_type, error_failure) - expected_entries = self.response['values'] - received_entry = response['value'] + def __maybe_save_as(self, response, config, result): + check_type = PostProcessCheckType.SAVE_AS_VARIABLE + error_success = 'The test save the value "{value}" as {name}.' + error_name_does_not_exist = 'The test expects a value named "{name}" but it does not exists in the response."' - for expected_entry in expected_entries: - if not 'saveAs' in expected_entry: + for value in self.response['values']: + if not 'saveAs' in value: continue - saveAs = expected_entry['saveAs'] - if type(expected_entry) is dict and type(received_entry) is dict: - expected_name = expected_entry['name'] - if not expected_name in received_entry: - result.error( - PostProcessCheckType.SAVE_AS_VARIABLE, f'The test expects a value named "{expected_name}" but it does not exists in the response."') + expected_name = 'value' + received_value = response.get('value') + if not self.is_attribute: + expected_name = value.get('name') + if not expected_name in received_value: + result.error(check_type, error_name_does_not_exist.format(name=expected_name)) continue - received_value = received_entry[expected_name] - config[saveAs] = received_value - result.success(PostProcessCheckType.SAVE_AS_VARIABLE, f'The test save the value "{received_value}" as {saveAs}.') - else: - received_value = received_entry - config[saveAs] = received_value - result.success(PostProcessCheckType.SAVE_AS_VARIABLE, f'The test save the value "{received_value}" as {saveAs}.') + received_value = received_value.get(expected_name) if received_value else None + + save_as = value.get('saveAs') + config[save_as] = received_value + result.success(check_type, error_success.format(value=received_value, name=save_as)) def __as_mapping(self, definitions, cluster_name, target_name): element = definitions.get_type_by_name(cluster_name, target_name) @@ -508,34 +538,25 @@ def __init__(self, test_file, pics_file, definitions): self.tests = YamlTests(tests, self.__update_placeholder) def __update_placeholder(self, data): - data.arguments = self.__encode_values(data.arguments) - data.response = self.__encode_values(data.response) + self.__encode_values(data.arguments) + self.__encode_values(data.response) return data def __encode_values(self, container): if not container: - return None - - # TODO this should likely be moved to end of TestStep.__init__ - if 'value' in container: - container['values'] = [{'name': 'value', 'value': container['value']}] - del container['value'] - - if 'values' in container: - for idx, item in enumerate(container['values']): - if 'value' in item: - container['values'][idx]['value'] = self.__config_variable_substitution(item['value']) - - # TODO this should likely be moved to end of TestStep.__init__. But depends on rationale - if 'saveAs' in container: - if not 'values' in container: - # TODO Currently very unclear why this corner case is needed. Would be nice to add - # information as to why this would happen. - container['values'] = [{}] - container['values'][0]['saveAs'] = container['saveAs'] - del container['saveAs'] + return - return container + values = container['values'] + + for idx, item in enumerate(values): + if 'value' in item: + values[idx]['value'] = self.__config_variable_substitution(item['value']) + + if 'constraints' in item: + for constraint, constraint_value in item['constraints'].items(): + values[idx]['constraints'][constraint] = self.__config_variable_substitution(constraint_value) + + container['values'] = values def __config_variable_substitution(self, value): if type(value) is list: From e36f6840d9ab09bdc2727d767c748381e91f9f5a Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Wed, 7 Dec 2022 15:08:28 +0000 Subject: [PATCH 11/23] Small fixes --- scripts/tests/yamltests/YamlParser.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index 5f5c131b272c52..cb749dc6f5c6c1 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -14,7 +14,6 @@ # limitations under the License. import yaml -import string from enum import Enum from .constraints import get_constraints @@ -422,7 +421,7 @@ def __as_mapping(self, definitions, cluster_name, target_name): return target_name - def __update_with_definition(self, container, mapping_type, config): + def __update_with_definition(self, container: dict, mapping_type, config: dict): if not container or not mapping_type: return container @@ -466,10 +465,13 @@ def __update_value_with_definition(self, value, mapping_type, config): else: mapping = mapping_type[key] rv[key] = self.__update_value_with_definition(value[key], mapping, config) - value = rv - elif type(value) is list: - value = [self.__update_value_with_definition(entry, mapping_type, config) for entry in value] - elif value and not value in config: + return rv + if type(value) is list: + return [self.__update_value_with_definition(entry, mapping_type, config) for entry in value] + # TODO currently I am unsure if the check of `value not in config` is sufficant. For + # example let's say value = 'foo + 1' and map type is 'int64u', we would arguably do + # the wrong thing below. + if value is not None and value not in config: if mapping_type == 'int64u' or mapping_type == 'int64s' or mapping_type == 'bitmap64' or mapping_type == 'epoch_us': value = YamlFixes.try_apply_yaml_cpp_longlong_limitation_fix(value) value = YamlFixes.try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value) @@ -513,7 +515,7 @@ def __next__(self): class YamlParser: name: None PICS: None - # TODO config should be internal (_config). Also can it actually be None, or + # TODO config should be internal (_config) config: dict = {} tests: None From 310122b4223cbc3063b302516391356e8bcc81d7 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Wed, 7 Dec 2022 17:46:11 +0100 Subject: [PATCH 12/23] [Yaml Parser] Moves _convert_single_value_to_values up one block such that update_with_definition can be simplified --- scripts/tests/yamltests/YamlParser.py | 68 ++++++++++++--------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index cb749dc6f5c6c1..951302900e6b12 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -210,25 +210,26 @@ def __init__(self, test: dict, config: dict, definitions): _check_valid_keys(self.arguments, _TEST_ARGUMENTS_SECTION) _check_valid_keys(self.response, _TEST_RESPONSE_SECTION) + self._convert_single_value_to_values(self.arguments) + self._convert_single_value_to_values(self.response) + + argument_mapping = None + response_mapping = None + if self.is_attribute: attribute = definitions.get_attribute_by_name(self.cluster, self.attribute) if attribute: - attribute_definition = self.__as_mapping(definitions, self.cluster, attribute.definition.data_type.name) - self.arguments = self.__update_with_definition(self.arguments, attribute_definition, config) - self.response = self.__update_with_definition(self.response, attribute_definition, config) + attribute_mapping = self.__as_mapping(definitions, self.cluster, attribute.definition.data_type.name) + argument_mapping = attribute_mapping + response_mapping = attribute_mapping else: command = definitions.get_command_by_name(self.cluster, self.command) if command: - command_definition = self.__as_mapping(definitions, self.cluster, command.input_param) - response_definition = self.__as_mapping(definitions, self.cluster, command.output_param) - - self.arguments = self.__update_with_definition(self.arguments, command_definition, config) - self.response = self.__update_with_definition(self.response, response_definition, config) + argument_mapping = self.__as_mapping(definitions, self.cluster, command.input_param) + response_mapping = self.__as_mapping(definitions, self.cluster, command.output_param) - # TODO Move this calls before the previous blocks that calls update_with_definition to make the - # underlying update_with_definition logic simpler to read. - self._convert_single_value_to_values(self.arguments) - self._convert_single_value_to_values(self.response) + self.__update_with_definition(self.arguments, argument_mapping, config) + self.__update_with_definition(self.response, response_mapping, config) def _convert_single_value_to_values(self, container): if container is None or 'values' in container: @@ -423,33 +424,22 @@ def __as_mapping(self, definitions, cluster_name, target_name): def __update_with_definition(self, container: dict, mapping_type, config: dict): if not container or not mapping_type: - return container - - for key, items in container.items(): - if type(items) is list and key == 'values': - rv = [] - for item in items: - newItem = {} - for item_key in item: - if item_key == 'value': - newItem[item_key] = self.__update_value_with_definition( - item['value'], mapping_type[item['name']], config) - else: - if item_key == 'saveAs' and not item[item_key] in config: - config[item[item_key]] = None - newItem[item_key] = item[item_key] - rv.append(newItem) - container[key] = rv - elif key == 'value' or key == 'values': - if 'saveAs' in container and not container['saveAs'] in config: - config[container['saveAs']] = None - rv = self.__update_value_with_definition(items, mapping_type, config) - container[key] = rv - elif key == 'constraints': - for constraint in container[key]: - container[key][constraint] = self.__update_value_with_definition( - container[key][constraint], mapping_type, config) - return container + return + + for value in list(container['values']): + for key, item_value in list(value.items()): + mapping = mapping_type if self.is_attribute else mapping_type[value['name']] + + if key == 'value': + value[key] = self.__update_value_with_definition(item_value, mapping, config) + elif key == 'saveAs' and type(item_value) is str and not item_value in config: + config[item_value] = None + elif key == 'constraints': + for constraint, constraint_value in item_value.items(): + value[key][constraint] = self.__update_value_with_definition(constraint_value, mapping_type, config) + else: + # This key, value pair does not rely on cluster specifications. + pass def __update_value_with_definition(self, value, mapping_type, config): if not mapping_type: From e6d39190026d6b58d70245c4472b859bd174e47c Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Wed, 7 Dec 2022 20:13:54 +0000 Subject: [PATCH 13/23] Move `__update_placeholder` to TestStep Also TestStep have internal pointer to config dictionary --- scripts/tests/yamltests/YamlParser.py | 163 +++++++++++++------------- 1 file changed, 84 insertions(+), 79 deletions(-) diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index 951302900e6b12..1c0cb8cb083baf 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -184,6 +184,8 @@ def __init__(self, test: dict, config: dict, definitions): if not self.is_enabled: return + self._config = config + _check_valid_keys(test, _TEST_SECTION) self.label = _valueOrNone(test, 'label') @@ -228,8 +230,8 @@ def __init__(self, test: dict, config: dict, definitions): argument_mapping = self.__as_mapping(definitions, self.cluster, command.input_param) response_mapping = self.__as_mapping(definitions, self.cluster, command.output_param) - self.__update_with_definition(self.arguments, argument_mapping, config) - self.__update_with_definition(self.response, response_mapping, config) + self.__update_with_definition(self.arguments, argument_mapping) + self.__update_with_definition(self.response, response_mapping) def _convert_single_value_to_values(self, container): if container is None or 'values' in container: @@ -255,7 +257,7 @@ def _convert_single_value_to_values(self, container): container['values'] = [value] - def post_process_response(self, response, config): + def post_process_response(self, response): result = PostProcessResponseResult() if not self.__maybe_check_optional(response, result): @@ -264,9 +266,9 @@ def post_process_response(self, response, config): self.__maybe_check_error(response, result) if self.response: self.__maybe_check_cluster_error(response, result) - self.__maybe_check_values(response, config, result) - self.__maybe_check_constraints(response, config, result) - self.__maybe_save_as(response, config, result) + self.__maybe_check_values(response, result) + self.__maybe_check_constraints(response, result) + self.__maybe_save_as(response, result) return result @@ -333,7 +335,7 @@ def __maybe_check_cluster_error(self, response, result): # Nothing is logged here to not be redundant with the generic error checking code. pass - def __maybe_check_values(self, response, config, result): + def __maybe_check_values(self, response, result): check_type = PostProcessCheckType.RESPONSE_VALIDATION error_success = 'The test expectation "{name} == {value}" is true' error_failure = 'The test expectation "{name} == {value}" is false' @@ -360,7 +362,7 @@ def __maybe_check_values(self, response, config, result): else: result.error(check_type, error_failure.format(name=expected_name, value=expected_value)) - def __maybe_check_constraints(self, response, config, result): + def __maybe_check_constraints(self, response, result): check_type = PostProcessCheckType.CONSTRAINT_VALIDATION error_success = 'Constraints check passed' error_failure = 'Constraints check failed' @@ -387,7 +389,7 @@ def __maybe_check_constraints(self, response, config, result): # TODO would be helpful to be more verbose here result.error(check_type, error_failure) - def __maybe_save_as(self, response, config, result): + def __maybe_save_as(self, response, result): check_type = PostProcessCheckType.SAVE_AS_VARIABLE error_success = 'The test save the value "{value}" as {name}.' error_name_does_not_exist = 'The test expects a value named "{name}" but it does not exists in the response."' @@ -407,7 +409,7 @@ def __maybe_save_as(self, response, config, result): received_value = received_value.get(expected_name) if received_value else None save_as = value.get('saveAs') - config[save_as] = received_value + self._config[save_as] = received_value result.success(check_type, error_success.format(value=received_value, name=save_as)) def __as_mapping(self, definitions, cluster_name, target_name): @@ -422,7 +424,7 @@ def __as_mapping(self, definitions, cluster_name, target_name): return target_name - def __update_with_definition(self, container: dict, mapping_type, config: dict): + def __update_with_definition(self, container: dict, mapping_type): if not container or not mapping_type: return @@ -431,17 +433,17 @@ def __update_with_definition(self, container: dict, mapping_type, config: dict): mapping = mapping_type if self.is_attribute else mapping_type[value['name']] if key == 'value': - value[key] = self.__update_value_with_definition(item_value, mapping, config) - elif key == 'saveAs' and type(item_value) is str and not item_value in config: - config[item_value] = None + value[key] = self.__update_value_with_definition(item_value, mapping) + elif key == 'saveAs' and type(item_value) is str and not item_value in self._config: + self._config[item_value] = None elif key == 'constraints': for constraint, constraint_value in item_value.items(): - value[key][constraint] = self.__update_value_with_definition(constraint_value, mapping_type, config) + value[key][constraint] = self.__update_value_with_definition(constraint_value, mapping_type) else: # This key, value pair does not rely on cluster specifications. pass - def __update_value_with_definition(self, value, mapping_type, config): + def __update_value_with_definition(self, value, mapping_type): if not mapping_type: return value @@ -454,14 +456,14 @@ def __update_value_with_definition(self, value, mapping_type, config): rv[key] = value[key] # int64u else: mapping = mapping_type[key] - rv[key] = self.__update_value_with_definition(value[key], mapping, config) + rv[key] = self.__update_value_with_definition(value[key], mapping) return rv if type(value) is list: - return [self.__update_value_with_definition(entry, mapping_type, config) for entry in value] + return [self.__update_value_with_definition(entry, mapping_type) for entry in value] # TODO currently I am unsure if the check of `value not in config` is sufficant. For # example let's say value = 'foo + 1' and map type is 'int64u', we would arguably do # the wrong thing below. - if value is not None and value not in config: + if value is not None and value not in self._config: if mapping_type == 'int64u' or mapping_type == 'int64s' or mapping_type == 'bitmap64' or mapping_type == 'epoch_us': value = YamlFixes.try_apply_yaml_cpp_longlong_limitation_fix(value) value = YamlFixes.try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value) @@ -474,66 +476,17 @@ def __update_value_with_definition(self, value, mapping_type, config): return value + def update_placeholder(self): + '''Performs any config/variable substitutions to the TestStep's arguments and response. -class YamlTests: - __tests: None - __hook: None - __index: 0 - - count: 0 - - def __init__(self, tests, hook): - self.__tests = tests - self.__hook = hook - self.__index = 0 - - self.count = len(tests) - - def __iter__(self): + Intended to only be called by YamlTests, so that previously processed test steps that give + us the variable are substitued to the current TestStep before it is given to a runner to + run. + ''' + self.__encode_values(self.arguments) + self.__encode_values(self.response) return self - def __next__(self): - if self.__index < self.count: - data = self.__tests[self.__index] - data = self.__hook(data) - self.__index += 1 - return data - - raise StopIteration - - -class YamlParser: - name: None - PICS: None - # TODO config should be internal (_config) - config: dict = {} - tests: None - - def __init__(self, test_file, pics_file, definitions): - # TODO Needs supports for PICS file - with open(test_file) as f: - loader = yaml.FullLoader - loader = YamlFixes.try_add_yaml_support_for_scientific_notation_without_dot(loader) - - data = yaml.load(f, Loader=loader) - _check_valid_keys(data, _TESTS_SECTION) - - self.name = _valueOrNone(data, 'name') - self.PICS = _valueOrNone(data, 'PICS') - - self.config = _valueOrNone(data, 'config') - - tests = list(filter(lambda test: test.is_enabled, [TestStep( - test, self.config, definitions) for test in _valueOrNone(data, 'tests')])) - YamlFixes.try_update_yaml_node_id_test_runner_state(tests, self.config) - - self.tests = YamlTests(tests, self.__update_placeholder) - - def __update_placeholder(self, data): - self.__encode_values(data.arguments) - self.__encode_values(data.response) - return data - def __encode_values(self, container): if not container: return @@ -569,8 +522,8 @@ def __config_variable_substitution(self, value): substitution_occured = False for idx, token in enumerate(tokens): - if token in self.config: - variable_info = self.config[token] + if token in self._config: + variable_info = self._config[token] if type(variable_info) is dict and 'defaultValue' in variable_info: variable_info = variable_info['defaultValue'] tokens[idx] = variable_info @@ -587,5 +540,57 @@ def __config_variable_substitution(self, value): else: return value + +class YamlTests: + __tests: list[TestStep] + __index: 0 + + count: 0 + + def __init__(self, tests: list[TestStep]): + self.__tests = tests + self.__index = 0 + + self.count = len(tests) + + def __iter__(self): + return self + + def __next__(self): + if self.__index < self.count: + test = self.__tests[self.__index] + test.update_placeholder() + self.__index += 1 + return test + + raise StopIteration + + +class YamlParser: + name: None + PICS: None + tests: None + _config: dict = {} + + def __init__(self, test_file, pics_file, definitions): + # TODO Needs supports for PICS file + with open(test_file) as f: + loader = yaml.FullLoader + loader = YamlFixes.try_add_yaml_support_for_scientific_notation_without_dot(loader) + + data = yaml.load(f, Loader=loader) + _check_valid_keys(data, _TESTS_SECTION) + + self.name = _valueOrNone(data, 'name') + self.PICS = _valueOrNone(data, 'PICS') + + self._config = _valueOrNone(data, 'config') + + tests = list(filter(lambda test: test.is_enabled, [TestStep( + test, self._config, definitions) for test in _valueOrNone(data, 'tests')])) + YamlFixes.try_update_yaml_node_id_test_runner_state(tests, self._config) + + self.tests = YamlTests(tests) + def update_config(self, key, value): - self.config[key] = value + self._config[key] = value From 6eba1462514144b46cf501fea015d238ebf6770d Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Mon, 12 Dec 2022 14:23:37 +0000 Subject: [PATCH 14/23] Add docstring, and initial constraint parsing --- scripts/tests/yamltests/YamlParser.py | 45 ++++++++++++++++++++------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index 1c0cb8cb083baf..e0bc6315e93787 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -78,15 +78,16 @@ class PostProcessCheckStatus(Enum): + ''' Inidcates the post processing check step status.''' SUCCESS = 'success', WARNING = 'warning', ERROR = 'error' -# TODO these values are just what was there before, these should be updated class PostProcessCheckType(Enum): - IM_STATUS = 'Error', - CLUSTER_STATUS = 'ClusterError', + ''' Inidcates the post processing check step type.''' + IM_STATUS = 'IMStatus', + CLUSTER_STATUS = 'ClusterStatus', RESPONSE_VALIDATION = 'Response', CONSTRAINT_VALIDATION = 'Constraints', SAVE_AS_VARIABLE = 'SaveAs' @@ -99,6 +100,12 @@ class PostProcessCheckType(Enum): # * failure: Indicates that the check was unsuccessfull # * warning: Indicates that the check is probably successful but that something needs to be considered. class PostProcessCheck: + ''' Information about a single post processing on check step. + + Each check has a helpful message should the consumer want to closer inspect what checks are, + as well as details as to why something may have failed. + ''' + def __init__(self, state: PostProcessCheckStatus, category: PostProcessCheckType, message: str): self.state = state self.category = category @@ -115,10 +122,11 @@ def is_error(self) -> bool: class PostProcessResponseResult: - ''' asdf + ''' Post processing response result information. - There are multiple steps that occur when post processing a response. This is a summary of the - results. Note that the number and types of steps performed is dependant on test step itself. + There are multiple steps that occur when post processing a response. This contains all the + results for each step performed. Note that the number and types of steps performed is + dependant on test step itself. ''' def __init__(self): @@ -129,17 +137,17 @@ def __init__(self): def success(self, category: PostProcessCheckType, message: str): ''' Adds a success entry that occured when post processing response to results.''' - self.__insert(PostProcessCheckStatus.SUCCESS, category, message) + self._insert(PostProcessCheckStatus.SUCCESS, category, message) self.successes += 1 def warning(self, category: PostProcessCheckType, message: str): ''' Adds a warning entry that occured when post processing response to results.''' - self.__insert(PostProcessCheckStatus.WARNING, category, message) + self._insert(PostProcessCheckStatus.WARNING, category, message) self.warnings += 1 def error(self, category: PostProcessCheckType, message: str): ''' Adds an error entry that occured when post processing response to results.''' - self.__insert(PostProcessCheckStatus.ERROR, category, message) + self._insert(PostProcessCheckStatus.ERROR, category, message) self.errors += 1 def is_success(self): @@ -150,7 +158,7 @@ def is_success(self): def is_failure(self): return self.errors != 0 - def __insert(self, state: PostProcessCheckStatus, category: PostProcessCheckType, message: str): + def _insert(self, state: PostProcessCheckStatus, category: PostProcessCheckType, message: str): log = PostProcessCheck(state, category, message) self.entries.append(log) @@ -172,6 +180,10 @@ def _valueOrConfig(data, key, config): class TestStep: + '''A single yaml test action parsed from YAML. + + There are stages to the lifecycle of this object. + ''' is_enabled: True is_command: False is_attribute: False @@ -233,6 +245,17 @@ def __init__(self, test: dict, config: dict, definitions): self.__update_with_definition(self.arguments, argument_mapping) self.__update_with_definition(self.response, response_mapping) + # This performs a very basic sanity parse time check of constrains. This parsing happens + # again inside post processing response since at that time we will have required variables + # to substitute in. This parsing check here has value since some test can take a really long + # time to run so knowing earlier on that the test step would have failed at parsing time + # before the test step run occurs save developer time that building yaml tests. + if self.response: + for value in self.response['values']: + if not 'constraints' in value: + continue + get_constraints(value['constraints']) + def _convert_single_value_to_values(self, container): if container is None or 'values' in container: return @@ -316,7 +339,7 @@ def __maybe_check_error(self, response, result): raise AssertionError('This should not happens.') def __maybe_check_cluster_error(self, response, result): - check_type = PostProcessCheckType.IM_STATUS + check_type = PostProcessCheckType.CLUSTER_STATUS error_success = 'The test expects the "{error}" error which occured successfully.' error_unexpected_success = 'The test expects the "{error}" error but no error occured.' error_wrong_error = 'The test expects the "{error}" error but the "{value}" error occured.' From 06218a36747083a1805843c0d3f9a5aadb88da0f Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Mon, 12 Dec 2022 16:00:49 +0000 Subject: [PATCH 15/23] Refactor for TestStep into two parts As well as stype formatting to get it ready for review --- scripts/tests/yamltests/YamlParser.py | 470 +++++++++++++++----------- 1 file changed, 282 insertions(+), 188 deletions(-) diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/YamlParser.py index e0bc6315e93787..75efa62f6bde83 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/YamlParser.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import yaml from enum import Enum @@ -93,12 +94,6 @@ class PostProcessCheckType(Enum): SAVE_AS_VARIABLE = 'SaveAs' -# Each 'check' add an entry into the logs db. This entry contains the success of failure state as well as a log -# message describing the check itself. -# A check state can be any of the three valid state: -# * success: Indicates that the check was successfull -# * failure: Indicates that the check was unsuccessfull -# * warning: Indicates that the check is probably successful but that something needs to be considered. class PostProcessCheck: ''' Information about a single post processing on check step. @@ -166,28 +161,30 @@ def _insert(self, state: PostProcessCheckStatus, category: PostProcessCheckType, def _check_valid_keys(section, valid_keys_dict): if section: for key in section: - if not key in valid_keys_dict: + if key not in valid_keys_dict: print(f'Unknown key: {key}') raise KeyError -def _valueOrNone(data, key): +def _value_or_none(data, key): return data[key] if key in data else None -def _valueOrConfig(data, key, config): +def _value_or_config(data, key, config): return data[key] if key in data else config[key] -class TestStep: - '''A single yaml test action parsed from YAML. +class _TestStepWithPlaceholders: + '''A single YAML test parsed, as is, from YAML. - There are stages to the lifecycle of this object. + Some YAML test steps contain placeholders for variable subsitution. The value of the variable + is only known after an earlier test step's has executed and the result successfully post + processed. ''' - is_enabled: True - is_command: False - is_attribute: False - is_event: False + is_enabled = True + is_command = False + is_attribute = False + is_event = False def __init__(self, test: dict, config: dict, definitions): # Disabled tests are not parsed in order to allow the test to be added to the test @@ -196,36 +193,36 @@ def __init__(self, test: dict, config: dict, definitions): if not self.is_enabled: return - self._config = config + self._parsing_config_variable_storage = config _check_valid_keys(test, _TEST_SECTION) - self.label = _valueOrNone(test, 'label') - self.optional = _valueOrNone(test, 'optional') - self.nodeId = _valueOrConfig(test, 'nodeId', config) - self.cluster = _valueOrConfig(test, 'cluster', config) - self.command = _valueOrConfig(test, 'command', config) - self.attribute = _valueOrNone(test, 'attribute') - self.endpoint = _valueOrConfig(test, 'endpoint', config) - - self.identity = _valueOrNone(test, 'identity') - self.fabricFiltered = _valueOrNone(test, 'fabricFiltered') - self.minInterval = _valueOrNone(test, 'minInterval') - self.maxInterval = _valueOrNone(test, 'maxInterval') - self.timedInteractionTimeoutMs = _valueOrNone(test, 'timedInteractionTimeoutMs') - self.busyWaitMs = _valueOrNone(test, 'busyWaitMs') + self.label = _value_or_none(test, 'label') + self.optional = _value_or_none(test, 'optional') + self.nodeId = _value_or_config(test, 'nodeId', config) + self.cluster = _value_or_config(test, 'cluster', config) + self.command = _value_or_config(test, 'command', config) + self.attribute = _value_or_none(test, 'attribute') + self.endpoint = _value_or_config(test, 'endpoint', config) + + self.identity = _value_or_none(test, 'identity') + self.fabricFiltered = _value_or_none(test, 'fabricFiltered') + self.minInterval = _value_or_none(test, 'minInterval') + self.maxInterval = _value_or_none(test, 'maxInterval') + self.timedInteractionTimeoutMs = _value_or_none(test, 'timedInteractionTimeoutMs') + self.busyWaitMs = _value_or_none(test, 'busyWaitMs') self.is_attribute = self.command in _ATTRIBUTE_COMMANDS self.is_event = self.command in _EVENT_COMMANDS - self.arguments = _valueOrNone(test, 'arguments') - self.response = _valueOrNone(test, 'response') + self.arguments_with_placeholders = _value_or_none(test, 'arguments') + self.response_with_placeholders = _value_or_none(test, 'response') - _check_valid_keys(self.arguments, _TEST_ARGUMENTS_SECTION) - _check_valid_keys(self.response, _TEST_RESPONSE_SECTION) + _check_valid_keys(self.arguments_with_placeholders, _TEST_ARGUMENTS_SECTION) + _check_valid_keys(self.response_with_placeholders, _TEST_RESPONSE_SECTION) - self._convert_single_value_to_values(self.arguments) - self._convert_single_value_to_values(self.response) + self._convert_single_value_to_values(self.arguments_with_placeholders) + self._convert_single_value_to_values(self.response_with_placeholders) argument_mapping = None response_mapping = None @@ -233,26 +230,27 @@ def __init__(self, test: dict, config: dict, definitions): if self.is_attribute: attribute = definitions.get_attribute_by_name(self.cluster, self.attribute) if attribute: - attribute_mapping = self.__as_mapping(definitions, self.cluster, attribute.definition.data_type.name) + attribute_mapping = self._as_mapping(definitions, self.cluster, + attribute.definition.data_type.name) argument_mapping = attribute_mapping response_mapping = attribute_mapping else: command = definitions.get_command_by_name(self.cluster, self.command) if command: - argument_mapping = self.__as_mapping(definitions, self.cluster, command.input_param) - response_mapping = self.__as_mapping(definitions, self.cluster, command.output_param) + argument_mapping = self._as_mapping(definitions, self.cluster, command.input_param) + response_mapping = self._as_mapping(definitions, self.cluster, command.output_param) - self.__update_with_definition(self.arguments, argument_mapping) - self.__update_with_definition(self.response, response_mapping) + self._update_with_definition(self.arguments_with_placeholders, argument_mapping) + self._update_with_definition(self.response_with_placeholders, response_mapping) # This performs a very basic sanity parse time check of constrains. This parsing happens # again inside post processing response since at that time we will have required variables - # to substitute in. This parsing check here has value since some test can take a really long - # time to run so knowing earlier on that the test step would have failed at parsing time - # before the test step run occurs save developer time that building yaml tests. - if self.response: - for value in self.response['values']: - if not 'constraints' in value: + # to substitute in. This parsing check here has value since some test can take a really + # long time to run so knowing earlier on that the test step would have failed at parsing + # time before the test step run occurs save developer time that building yaml tests. + if self.response_with_placeholders: + for value in self.response_with_placeholders['values']: + if 'constraints' not in value: continue get_constraints(value['constraints']) @@ -280,33 +278,192 @@ def _convert_single_value_to_values(self, container): container['values'] = [value] - def post_process_response(self, response): + def _as_mapping(self, definitions, cluster_name, target_name): + element = definitions.get_type_by_name(cluster_name, target_name) + + if hasattr(element, 'base_type'): + target_name = element.base_type.lower() + elif hasattr(element, 'fields'): + target_name = {f.name: self._as_mapping(definitions, cluster_name, f.data_type.name) for f in element.fields} + elif target_name: + target_name = target_name.lower() + + return target_name + + def _update_with_definition(self, container: dict, mapping_type): + if not container or not mapping_type: + return + + for value in list(container['values']): + for key, item_value in list(value.items()): + mapping = mapping_type if self.is_attribute else mapping_type[value['name']] + + if key == 'value': + value[key] = self._update_value_with_definition(item_value, mapping) + elif key == 'saveAs' and type(item_value) is str and item_value not in self._parsing_config_variable_storage: + self._parsing_config_variable_storage[item_value] = None + elif key == 'constraints': + for constraint, constraint_value in item_value.items(): + value[key][constraint] = self._update_value_with_definition( + constraint_value, mapping_type) + else: + # This key, value pair does not rely on cluster specifications. + pass + + def _update_value_with_definition(self, value, mapping_type): + if not mapping_type: + return value + + if type(value) is dict: + rv = {} + for key in value: + # FabricIndex is a special case where the framework requires it to be passed even + # if it is not part of the requested arguments per spec and not part of the XML + # definition. + if key == 'FabricIndex' or key == 'fabricIndex': + rv[key] = value[key] # int64u + else: + mapping = mapping_type[key] + rv[key] = self._update_value_with_definition(value[key], mapping) + return rv + if type(value) is list: + return [self._update_value_with_definition(entry, mapping_type) for entry in value] + # TODO currently I am unsure if the check of `value not in config` is sufficant. For + # example let's say value = 'foo + 1' and map type is 'int64u', we would arguably do + # the wrong thing below. + if value is not None and value not in self._parsing_config_variable_storage: + if mapping_type == 'int64u' or mapping_type == 'int64s' or mapping_type == 'bitmap64' or mapping_type == 'epoch_us': + value = YamlFixes.try_apply_yaml_cpp_longlong_limitation_fix(value) + value = YamlFixes.try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value) + elif mapping_type == 'single' or mapping_type == 'double': + value = YamlFixes.try_apply_yaml_float_written_as_strings(value) + elif mapping_type == 'octet_string' or mapping_type == 'long_octet_string': + value = YamlFixes.convert_yaml_octet_string_to_bytes(value) + elif mapping_type == 'boolean': + value = bool(value) + + return value + + +class TestStep: + '''A single YAML test action parsed from YAML. + + This object for the time being is somewhat stateful. When first created it contains + ''' + + def __init__(self, test: _TestStepWithPlaceholders, runtime_config_variable_storage: dict): + self._test = test + self._runtime_config_variable_storage = runtime_config_variable_storage + self.arguments = copy.deepcopy(test.arguments_with_placeholders) + self.response = copy.deepcopy(test.response_with_placeholders) + self._update_placeholder_values(self.arguments) + self._update_placeholder_values(self.response) + + @property + def is_enabled(self): + return self._test.is_enabled + + @property + def is_command(self): + return self._test.is_command + + @property + def is_attribute(self): + return self._test.is_attribute + + @property + def is_event(self): + return self._test.is_event + + @property + def label(self): + return self._test.label + + @property + def optional(self): + return self._test.optional + + @property + def nodeId(self): + return self._test.nodeId + + @property + def cluster(self): + return self._test.cluster + + @property + def command(self): + return self._test.command + + @property + def attribute(self): + return self._test.attribute + + @property + def endpoint(self): + return self._test.endpoint + + @property + def identity(self): + return self._test.identity + + @property + def fabricFiltered(self): + return self._test.fabricFiltered + + @property + def minInterval(self): + return self._test.minInterval + + @property + def maxInterval(self): + return self._test.maxInterval + + @property + def timedInteractionTimeoutMs(self): + return self._test.timedInteractionTimeoutMs + + @property + def busyWaitMs(self): + return self._test.busyWaitMs + + def post_process_response(self, response: dict): result = PostProcessResponseResult() - if not self.__maybe_check_optional(response, result): + if self._skip_post_processing(response, result): return result - self.__maybe_check_error(response, result) + self._response_error_validation(response, result) if self.response: - self.__maybe_check_cluster_error(response, result) - self.__maybe_check_values(response, result) - self.__maybe_check_constraints(response, result) - self.__maybe_save_as(response, result) + self._response_cluster_error_validation(response, result) + self._response_values_validation(response, result) + self._response_constraints_validation(response, result) + self._maybe_save_as(response, result) return result - def __maybe_check_optional(self, response, result): - if not self.optional or not 'error' in response: - return True + def _skip_post_processing(self, response: dict, result) -> bool: + ''' Should we skip perform post processing. - received_error = response.get('error') - if received_error == 'UNSUPPORTED_ATTRIBUTE' or received_error == 'UNSUPPORTED_COMMAND': - # result.warning('Optional', f'The response contains the error: "{error}".') + Currently we only skip post processing if the test step indicates that sent test step + invokation was expected to be optionally supported. We confirm that it is optional + supported by either validating we got the expected error only then indicate that all + other post processing should be skipped. + ''' + if not self.optional: + return False + + received_error = response.get('error', None) + if received_error is None: return False - return True + if received_error == 'UNSUPPORTED_ATTRIBUTE' or received_error == 'UNSUPPORTED_COMMAND': + # result.warning(PostProcessCheckType.Optional, f'The response contains the error: "{error}".') + return True - def __maybe_check_error(self, response, result): + return False + + def _response_error_validation(self, response, result): check_type = PostProcessCheckType.IM_STATUS error_success = 'The test expects the "{error}" error which occured successfully.' error_success_no_error = 'The test expects no error and no error occurred.' @@ -327,7 +484,8 @@ def __maybe_check_error(self, response, result): if expected_error and received_error and expected_error == received_error: result.success(check_type, error_success.format(error=expected_error)) elif expected_error and received_error: - result.error(check_type, error_wrong_error.format(error=expected_error, value=received_error)) + result.error(check_type, error_wrong_error.format( + error=expected_error, value=received_error)) elif expected_error and not received_error: result.error(check_type, error_unexpected_success.format(error=expected_error)) elif not expected_error and received_error: @@ -338,7 +496,7 @@ def __maybe_check_error(self, response, result): # This should not happens raise AssertionError('This should not happens.') - def __maybe_check_cluster_error(self, response, result): + def _response_cluster_error_validation(self, response, result): check_type = PostProcessCheckType.CLUSTER_STATUS error_success = 'The test expects the "{error}" error which occured successfully.' error_unexpected_success = 'The test expects the "{error}" error but no error occured.' @@ -351,28 +509,29 @@ def __maybe_check_cluster_error(self, response, result): if received_error and expected_error == received_error: result.success(check_type, error_success.format(error=expected_error)) elif received_error: - result.error(check_type, error_wrong_error.format(error=expected_error, value=received_error)) + result.error(check_type, error_wrong_error.format( + error=expected_error, value=received_error)) else: result.error(check_type, error_unexpected_success.format(error=expected_error)) else: # Nothing is logged here to not be redundant with the generic error checking code. pass - def __maybe_check_values(self, response, result): + def _response_values_validation(self, response, result): check_type = PostProcessCheckType.RESPONSE_VALIDATION error_success = 'The test expectation "{name} == {value}" is true' error_failure = 'The test expectation "{name} == {value}" is false' error_name_does_not_exist = 'The test expects a value named "{name}" but it does not exists in the response."' for value in self.response['values']: - if not 'value' in value: + if 'value' not in value: continue expected_name = 'value' received_value = response.get('value') if not self.is_attribute: expected_name = value.get('name') - if not expected_name in received_value: + if expected_name not in received_value: result.error(check_type, error_name_does_not_exist.format(name=expected_name)) continue @@ -381,25 +540,27 @@ def __maybe_check_values(self, response, result): # TODO Supports Array/List. See an exemple of failure in TestArmFailSafe.yaml expected_value = value.get('value') if expected_value == received_value: - result.success(check_type, error_success.format(name=expected_name, value=received_value)) + result.success( + check_type, error_success.format(name=expected_name, value=received_value)) else: - result.error(check_type, error_failure.format(name=expected_name, value=expected_value)) + result.error( + check_type, error_failure.format(name=expected_name, value=expected_value)) - def __maybe_check_constraints(self, response, result): + def _response_constraints_validation(self, response, result): check_type = PostProcessCheckType.CONSTRAINT_VALIDATION error_success = 'Constraints check passed' error_failure = 'Constraints check failed' error_name_does_not_exist = 'The test expects a value named "{name}" but it does not exists in the response."' for value in self.response['values']: - if not 'constraints' in value: + if 'constraints' not in value: continue expected_name = 'value' received_value = response.get('value') if not self.is_attribute: expected_name = value.get('name') - if not expected_name in received_value: + if expected_name not in received_value: result.error(check_type, error_name_does_not_exist.format(name=expected_name)) continue @@ -412,105 +573,30 @@ def __maybe_check_constraints(self, response, result): # TODO would be helpful to be more verbose here result.error(check_type, error_failure) - def __maybe_save_as(self, response, result): + def _maybe_save_as(self, response, result): check_type = PostProcessCheckType.SAVE_AS_VARIABLE error_success = 'The test save the value "{value}" as {name}.' error_name_does_not_exist = 'The test expects a value named "{name}" but it does not exists in the response."' for value in self.response['values']: - if not 'saveAs' in value: + if 'saveAs' not in value: continue expected_name = 'value' received_value = response.get('value') if not self.is_attribute: expected_name = value.get('name') - if not expected_name in received_value: + if expected_name not in received_value: result.error(check_type, error_name_does_not_exist.format(name=expected_name)) continue received_value = received_value.get(expected_name) if received_value else None save_as = value.get('saveAs') - self._config[save_as] = received_value + self._runtime_config_variable_storage[save_as] = received_value result.success(check_type, error_success.format(value=received_value, name=save_as)) - def __as_mapping(self, definitions, cluster_name, target_name): - element = definitions.get_type_by_name(cluster_name, target_name) - - if hasattr(element, 'base_type'): - target_name = element.base_type.lower() - elif hasattr(element, 'fields'): - target_name = {f.name: self.__as_mapping(definitions, cluster_name, f.data_type.name) for f in element.fields} - elif target_name: - target_name = target_name.lower() - - return target_name - - def __update_with_definition(self, container: dict, mapping_type): - if not container or not mapping_type: - return - - for value in list(container['values']): - for key, item_value in list(value.items()): - mapping = mapping_type if self.is_attribute else mapping_type[value['name']] - - if key == 'value': - value[key] = self.__update_value_with_definition(item_value, mapping) - elif key == 'saveAs' and type(item_value) is str and not item_value in self._config: - self._config[item_value] = None - elif key == 'constraints': - for constraint, constraint_value in item_value.items(): - value[key][constraint] = self.__update_value_with_definition(constraint_value, mapping_type) - else: - # This key, value pair does not rely on cluster specifications. - pass - - def __update_value_with_definition(self, value, mapping_type): - if not mapping_type: - return value - - if type(value) is dict: - rv = {} - for key in value: - # FabricIndex is a special case where the framework requires it to be passed even if it is not part of the - # requested arguments per spec and not part of the XML definition. - if key == 'FabricIndex' or key == 'fabricIndex': - rv[key] = value[key] # int64u - else: - mapping = mapping_type[key] - rv[key] = self.__update_value_with_definition(value[key], mapping) - return rv - if type(value) is list: - return [self.__update_value_with_definition(entry, mapping_type) for entry in value] - # TODO currently I am unsure if the check of `value not in config` is sufficant. For - # example let's say value = 'foo + 1' and map type is 'int64u', we would arguably do - # the wrong thing below. - if value is not None and value not in self._config: - if mapping_type == 'int64u' or mapping_type == 'int64s' or mapping_type == 'bitmap64' or mapping_type == 'epoch_us': - value = YamlFixes.try_apply_yaml_cpp_longlong_limitation_fix(value) - value = YamlFixes.try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value) - elif mapping_type == 'single' or mapping_type == 'double': - value = YamlFixes.try_apply_yaml_float_written_as_strings(value) - elif mapping_type == 'octet_string' or mapping_type == 'long_octet_string': - value = YamlFixes.convert_yaml_octet_string_to_bytes(value) - elif mapping_type == 'boolean': - value = bool(value) - - return value - - def update_placeholder(self): - '''Performs any config/variable substitutions to the TestStep's arguments and response. - - Intended to only be called by YamlTests, so that previously processed test steps that give - us the variable are substitued to the current TestStep before it is given to a runner to - run. - ''' - self.__encode_values(self.arguments) - self.__encode_values(self.response) - return self - - def __encode_values(self, container): + def _update_placeholder_values(self, container): if not container: return @@ -518,35 +604,36 @@ def __encode_values(self, container): for idx, item in enumerate(values): if 'value' in item: - values[idx]['value'] = self.__config_variable_substitution(item['value']) + values[idx]['value'] = self._config_variable_substitution(item['value']) if 'constraints' in item: for constraint, constraint_value in item['constraints'].items(): - values[idx]['constraints'][constraint] = self.__config_variable_substitution(constraint_value) + values[idx]['constraints'][constraint] = self._config_variable_substitution( + constraint_value) container['values'] = values - def __config_variable_substitution(self, value): + def _config_variable_substitution(self, value): if type(value) is list: - return [self.__config_variable_substitution(entry) for entry in value] + return [self._config_variable_substitution(entry) for entry in value] elif type(value) is dict: mapped_value = {} for key in value: - mapped_value[key] = self.__config_variable_substitution(value[key]) + mapped_value[key] = self._config_variable_substitution(value[key]) return mapped_value elif type(value) is str: # For most tests, a single config variable is used and it can be replaced as in. # But some other tests were relying on the fact that the expression was put 'as if' in - # the generated code and was resolved before being sent over the wire. For such expressions - # (e.g 'myVar + 1') we need to compute it before sending it over the wire. + # the generated code and was resolved before being sent over the wire. For such + # expressions (e.g 'myVar + 1') we need to compute it before sending it over the wire. tokens = value.split() if len(tokens) == 0: return value substitution_occured = False for idx, token in enumerate(tokens): - if token in self._config: - variable_info = self._config[token] + if token in self._runtime_config_variable_storage: + variable_info = self._runtime_config_variable_storage[token] if type(variable_info) is dict and 'defaultValue' in variable_info: variable_info = variable_info['defaultValue'] tokens[idx] = variable_info @@ -557,34 +644,44 @@ def __config_variable_substitution(self, value): tokens = [str(token) for token in tokens] value = ' '.join(tokens) - # TODO we should move away from eval. That will mean that we will need to do extra parsing, - # but it would be safer then just blindly running eval. + # TODO we should move away from eval. That will mean that we will need to do extra + # parsing, but it would be safer then just blindly running eval. return value if not substitution_occured else eval(value) else: return value class YamlTests: - __tests: list[TestStep] - __index: 0 + _tests: list[_TestStepWithPlaceholders] + _index: 0 count: 0 - def __init__(self, tests: list[TestStep]): - self.__tests = tests - self.__index = 0 - - self.count = len(tests) + def __init__(self, parsing_config_variable_storage: dict, definitions, tests: dict): + self._parsing_config_variable_storage = parsing_config_variable_storage + enabled_tests = [] + for test in tests: + test_with_placeholders = _TestStepWithPlaceholders( + test, self._parsing_config_variable_storage, definitions) + if test_with_placeholders.is_enabled: + enabled_tests.append(test_with_placeholders) + YamlFixes.try_update_yaml_node_id_test_runner_state( + enabled_tests, self._parsing_config_variable_storage) + + self._runtime_config_variable_storage = copy.deepcopy(parsing_config_variable_storage) + self._tests = enabled_tests + self._index = 0 + self.count = len(self._tests) def __iter__(self): return self - def __next__(self): - if self.__index < self.count: - test = self.__tests[self.__index] - test.update_placeholder() - self.__index += 1 - return test + def __next__(self) -> TestStep: + if self._index < self.count: + test = self._tests[self._index] + test_step = TestStep(test, self._runtime_config_variable_storage) + self._index += 1 + return test_step raise StopIteration @@ -593,7 +690,7 @@ class YamlParser: name: None PICS: None tests: None - _config: dict = {} + _parsing_config_variable_storage: dict = {} def __init__(self, test_file, pics_file, definitions): # TODO Needs supports for PICS file @@ -604,16 +701,13 @@ def __init__(self, test_file, pics_file, definitions): data = yaml.load(f, Loader=loader) _check_valid_keys(data, _TESTS_SECTION) - self.name = _valueOrNone(data, 'name') - self.PICS = _valueOrNone(data, 'PICS') - - self._config = _valueOrNone(data, 'config') + self.name = _value_or_none(data, 'name') + self.PICS = _value_or_none(data, 'PICS') - tests = list(filter(lambda test: test.is_enabled, [TestStep( - test, self._config, definitions) for test in _valueOrNone(data, 'tests')])) - YamlFixes.try_update_yaml_node_id_test_runner_state(tests, self._config) + self._parsing_config_variable_storage = _value_or_none(data, 'config') - self.tests = YamlTests(tests) + tests = _value_or_none(data, 'tests') + self.tests = YamlTests(self._parsing_config_variable_storage, definitions, tests) def update_config(self, key, value): - self._config[key] = value + self._parsing_config_variable_storage[key] = value From ad4fa77ecaed1a75479cd7560d61c2895c3a9b64 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Mon, 12 Dec 2022 23:19:26 +0000 Subject: [PATCH 16/23] Converge to PEP8 python format standard --- scripts/tests/test_yaml_parser.py | 6 +-- .../{SpecDefinitions.py => definitions.py} | 0 .../yamltests/{YamlFixes.py => fixes.py} | 0 .../yamltests/{YamlParser.py => parser.py} | 52 +++++++++---------- .../tests/yamltests/test_spec_definitions.py | 2 +- 5 files changed, 30 insertions(+), 30 deletions(-) rename scripts/tests/yamltests/{SpecDefinitions.py => definitions.py} (100%) rename scripts/tests/yamltests/{YamlFixes.py => fixes.py} (100%) rename scripts/tests/yamltests/{YamlParser.py => parser.py} (95%) diff --git a/scripts/tests/test_yaml_parser.py b/scripts/tests/test_yaml_parser.py index e9154b1fca6ec3..787e1a15e21e76 100644 --- a/scripts/tests/test_yaml_parser.py +++ b/scripts/tests/test_yaml_parser.py @@ -25,8 +25,8 @@ import functools from pathlib import Path -from yamltests.SpecDefinitions import * -from yamltests.YamlParser import YamlParser +from yamltests.definitions import * +from yamltests.parser import TestParser _DEFAULT_MATTER_ROOT = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..')) @@ -80,7 +80,7 @@ def setUp(self): sources = [ParseSource(source=name) for name in filenames] specifications = SpecDefinitions(sources) - self._yaml_parser = YamlParser(path_to_test, pics_file, specifications) + self._yaml_parser = TestParser(path_to_test, pics_file, specifications) def test_able_to_iterate_over_all_tests(self): # self._yaml_parser.tests implements `__next__`, which does value substitution. We are diff --git a/scripts/tests/yamltests/SpecDefinitions.py b/scripts/tests/yamltests/definitions.py similarity index 100% rename from scripts/tests/yamltests/SpecDefinitions.py rename to scripts/tests/yamltests/definitions.py diff --git a/scripts/tests/yamltests/YamlFixes.py b/scripts/tests/yamltests/fixes.py similarity index 100% rename from scripts/tests/yamltests/YamlFixes.py rename to scripts/tests/yamltests/fixes.py diff --git a/scripts/tests/yamltests/YamlParser.py b/scripts/tests/yamltests/parser.py similarity index 95% rename from scripts/tests/yamltests/YamlParser.py rename to scripts/tests/yamltests/parser.py index 75efa62f6bde83..7a1bd7705163fe 100644 --- a/scripts/tests/yamltests/YamlParser.py +++ b/scripts/tests/yamltests/parser.py @@ -18,7 +18,7 @@ from enum import Enum from .constraints import get_constraints -from . import YamlFixes +from . import fixes _TESTS_SECTION = [ 'name', @@ -199,18 +199,18 @@ def __init__(self, test: dict, config: dict, definitions): self.label = _value_or_none(test, 'label') self.optional = _value_or_none(test, 'optional') - self.nodeId = _value_or_config(test, 'nodeId', config) + self.node_id = _value_or_config(test, 'nodeId', config) self.cluster = _value_or_config(test, 'cluster', config) self.command = _value_or_config(test, 'command', config) self.attribute = _value_or_none(test, 'attribute') self.endpoint = _value_or_config(test, 'endpoint', config) self.identity = _value_or_none(test, 'identity') - self.fabricFiltered = _value_or_none(test, 'fabricFiltered') - self.minInterval = _value_or_none(test, 'minInterval') - self.maxInterval = _value_or_none(test, 'maxInterval') - self.timedInteractionTimeoutMs = _value_or_none(test, 'timedInteractionTimeoutMs') - self.busyWaitMs = _value_or_none(test, 'busyWaitMs') + self.fabric_filtered = _value_or_none(test, 'fabricFiltered') + self.min_interval = _value_or_none(test, 'minInterval') + self.max_interval = _value_or_none(test, 'maxInterval') + self.timed_interaction_timeout_ms = _value_or_none(test, 'timedInteractionTimeoutMs') + self.busy_wait_ms = _value_or_none(test, 'busyWaitMs') self.is_attribute = self.command in _ATTRIBUTE_COMMANDS self.is_event = self.command in _EVENT_COMMANDS @@ -333,12 +333,12 @@ def _update_value_with_definition(self, value, mapping_type): # the wrong thing below. if value is not None and value not in self._parsing_config_variable_storage: if mapping_type == 'int64u' or mapping_type == 'int64s' or mapping_type == 'bitmap64' or mapping_type == 'epoch_us': - value = YamlFixes.try_apply_yaml_cpp_longlong_limitation_fix(value) - value = YamlFixes.try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value) + value = fixes.try_apply_yaml_cpp_longlong_limitation_fix(value) + value = fixes.try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value) elif mapping_type == 'single' or mapping_type == 'double': - value = YamlFixes.try_apply_yaml_float_written_as_strings(value) + value = fixes.try_apply_yaml_float_written_as_strings(value) elif mapping_type == 'octet_string' or mapping_type == 'long_octet_string': - value = YamlFixes.convert_yaml_octet_string_to_bytes(value) + value = fixes.convert_yaml_octet_string_to_bytes(value) elif mapping_type == 'boolean': value = bool(value) @@ -384,8 +384,8 @@ def optional(self): return self._test.optional @property - def nodeId(self): - return self._test.nodeId + def node_id(self): + return self._test.node_id @property def cluster(self): @@ -408,24 +408,24 @@ def identity(self): return self._test.identity @property - def fabricFiltered(self): - return self._test.fabricFiltered + def fabric_filtered(self): + return self._test.fabric_filtered @property - def minInterval(self): - return self._test.minInterval + def min_interval(self): + return self._test.min_interval @property - def maxInterval(self): - return self._test.maxInterval + def max_interval(self): + return self._test.max_interval @property - def timedInteractionTimeoutMs(self): - return self._test.timedInteractionTimeoutMs + def timed_interaction_timeout_ms(self): + return self._test.timed_interaction_timeout_ms @property - def busyWaitMs(self): - return self._test.busyWaitMs + def busy_wait_ms(self): + return self._test.busy_wait_ms def post_process_response(self, response: dict): result = PostProcessResponseResult() @@ -665,7 +665,7 @@ def __init__(self, parsing_config_variable_storage: dict, definitions, tests: di test, self._parsing_config_variable_storage, definitions) if test_with_placeholders.is_enabled: enabled_tests.append(test_with_placeholders) - YamlFixes.try_update_yaml_node_id_test_runner_state( + fixes.try_update_yaml_node_id_test_runner_state( enabled_tests, self._parsing_config_variable_storage) self._runtime_config_variable_storage = copy.deepcopy(parsing_config_variable_storage) @@ -686,7 +686,7 @@ def __next__(self) -> TestStep: raise StopIteration -class YamlParser: +class TestParser: name: None PICS: None tests: None @@ -696,7 +696,7 @@ def __init__(self, test_file, pics_file, definitions): # TODO Needs supports for PICS file with open(test_file) as f: loader = yaml.FullLoader - loader = YamlFixes.try_add_yaml_support_for_scientific_notation_without_dot(loader) + loader = fixes.try_add_yaml_support_for_scientific_notation_without_dot(loader) data = yaml.load(f, Loader=loader) _check_valid_keys(data, _TESTS_SECTION) diff --git a/scripts/tests/yamltests/test_spec_definitions.py b/scripts/tests/yamltests/test_spec_definitions.py index fbf480c6beb083..00a15da9940d0c 100644 --- a/scripts/tests/yamltests/test_spec_definitions.py +++ b/scripts/tests/yamltests/test_spec_definitions.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from SpecDefinitions import * +from definitions import * import unittest import io From a4067e0b03e084593867c3ce930641b6765bdd7f Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Tue, 13 Dec 2022 14:37:33 +0000 Subject: [PATCH 17/23] Add and clean up doc strings --- scripts/tests/yamltests/fixes.py | 40 +++++++++++++++++++------------ scripts/tests/yamltests/parser.py | 31 +++++++++++++----------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/scripts/tests/yamltests/fixes.py b/scripts/tests/yamltests/fixes.py index f8e3b54a0aaee8..a0126b003f9a2e 100644 --- a/scripts/tests/yamltests/fixes.py +++ b/scripts/tests/yamltests/fixes.py @@ -16,38 +16,51 @@ import re import binascii -# Some of the YAML files contains some values that has been crafted to avoid some limitations -# of the original JavaScript parser and the C++ generated bits. -# -# This class search the YAML for those changes and convert them back to something agnostic. +'''Fixes certain value formats known to exist in YAML tests + +Some of the YAML test files contains some values that has been crafted to avoid some limitations +of the original JavaScript parser and the C++ generated bits. This file contains functions to +changes and convert those YAML values them back to something agnostic. +''' def try_apply_yaml_cpp_longlong_limitation_fix(value): - # TestCluster contains a nasty hack for C++ in order to avoid a warning: -9223372036854775808 is not a valid way to write a long long in C++ it uses - # "-9223372036854775807LL - 1". This fix replace this hack by -9223372036854775808. + '''Fix a known nasty hack in order that was used to work around compiler issue for C++. + + When -9223372036854775808 was provided compiler would give a warning saying it is not a valid + way to write a long long in C++. + ''' if value == "-9223372036854775807LL - 1": value = -9223372036854775808 return value def try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value): - # JavaScript can not represent integers bigger than 9007199254740991. But some of the test may uses values that are bigger - # than this. The current way to workaround this limitation has been to write those numbers as strings by encapsulating them in "". + '''Fix up large integers that are represented within a string. + + JavaScript can not represent integers bigger than 9007199254740991. But some of the test may + uses values that are bigger than this. The current way to workaround this limitation has + been to write those numbers as strings by encapsulating them in "". + ''' if type(value) is str: value = int(value) return value def try_apply_yaml_float_written_as_strings(value): + '''Fix up floats that are represented within a string.''' if type(value) is str: value = float(value) return value +# TODO(thampson) This method is a clone of the method in +# src/controller/python/chip/yaml/format_converter.py and should eventually be removed in that file. def convert_yaml_octet_string_to_bytes(s: str) -> bytes: - # This method is a clone of the method in src/controller/python/chip/yaml/format_converter.py - # It needs to be only a single copy of this method. - """Convert YAML octet string body to bytes, handling any c-style hex escapes (e.g. \x5a) and hex: prefix""" + '''Convert YAML octet string body to bytes. + + This handles any c-style hex escapes (e.g. \x5a) and hex: prefix + ''' # Step 1: handle explicit "hex:" prefix if s.startswith('hex:'): return binascii.unhexlify(s[4:]) @@ -64,8 +77,6 @@ def convert_yaml_octet_string_to_bytes(s: str) -> bytes: accumulated_hex = ''.join([f"{(v & 0xFF):02x}" for v in as_bytes]) return binascii.unhexlify(accumulated_hex) -# The PyYAML implementation does not match float according to the JSON and fails on valid numbers. - def try_add_yaml_support_for_scientific_notation_without_dot(loader): regular_expression = re.compile(u'''^(?: @@ -82,11 +93,10 @@ def try_add_yaml_support_for_scientific_notation_without_dot(loader): list(u'-+0123456789.')) return loader + # This is a gross hack. The previous runner has a some internal states where an identity match one # accessory. But this state may not exist in the runner (as in it prevent to have multiple node ids # associated to a fabric...) so the 'nodeId' needs to be added back manually. - - def try_update_yaml_node_id_test_runner_state(tests, config): identities = {'alpha': None if 'nodeId' not in config else config['nodeId']} diff --git a/scripts/tests/yamltests/parser.py b/scripts/tests/yamltests/parser.py index 7a1bd7705163fe..1f8b8a63321cff 100644 --- a/scripts/tests/yamltests/parser.py +++ b/scripts/tests/yamltests/parser.py @@ -79,14 +79,14 @@ class PostProcessCheckStatus(Enum): - ''' Inidcates the post processing check step status.''' + '''Inidcates the post processing check step status.''' SUCCESS = 'success', WARNING = 'warning', ERROR = 'error' class PostProcessCheckType(Enum): - ''' Inidcates the post processing check step type.''' + '''Inidcates the post processing check step type.''' IM_STATUS = 'IMStatus', CLUSTER_STATUS = 'ClusterStatus', RESPONSE_VALIDATION = 'Response', @@ -95,10 +95,10 @@ class PostProcessCheckType(Enum): class PostProcessCheck: - ''' Information about a single post processing on check step. + '''Information about a single post processing operation that was performed. - Each check has a helpful message should the consumer want to closer inspect what checks are, - as well as details as to why something may have failed. + Each check has a helpful message, indicating what the post processing operation did and whether + it was successful or not. ''' def __init__(self, state: PostProcessCheckStatus, category: PostProcessCheckType, message: str): @@ -117,10 +117,10 @@ def is_error(self) -> bool: class PostProcessResponseResult: - ''' Post processing response result information. + '''Post processing response result information. - There are multiple steps that occur when post processing a response. This contains all the - results for each step performed. Note that the number and types of steps performed is + There are multiple operations that occur when post processing a response. This contains all the + results for each operation performed. Note that the number and types of steps performed is dependant on test step itself. ''' @@ -131,17 +131,17 @@ def __init__(self): self.errors = 0 def success(self, category: PostProcessCheckType, message: str): - ''' Adds a success entry that occured when post processing response to results.''' + '''Adds a success entry that occured when post processing response to results.''' self._insert(PostProcessCheckStatus.SUCCESS, category, message) self.successes += 1 def warning(self, category: PostProcessCheckType, message: str): - ''' Adds a warning entry that occured when post processing response to results.''' + '''Adds a warning entry that occured when post processing response to results.''' self._insert(PostProcessCheckStatus.WARNING, category, message) self.warnings += 1 def error(self, category: PostProcessCheckType, message: str): - ''' Adds an error entry that occured when post processing response to results.''' + '''Adds an error entry that occured when post processing response to results.''' self._insert(PostProcessCheckStatus.ERROR, category, message) self.errors += 1 @@ -328,7 +328,7 @@ def _update_value_with_definition(self, value, mapping_type): return rv if type(value) is list: return [self._update_value_with_definition(entry, mapping_type) for entry in value] - # TODO currently I am unsure if the check of `value not in config` is sufficant. For + # TODO currently unsure if the check of `value not in config` is sufficant. For # example let's say value = 'foo + 1' and map type is 'int64u', we would arguably do # the wrong thing below. if value is not None and value not in self._parsing_config_variable_storage: @@ -348,7 +348,10 @@ def _update_value_with_definition(self, value, mapping_type): class TestStep: '''A single YAML test action parsed from YAML. - This object for the time being is somewhat stateful. When first created it contains + This object contains all the information required for a test runner to execute the test step. + It also provide a function that is expected to be called by the test runner to post process + the recieved response from the accessory. Post processing both validates recieved response + and saves any variables that might be required but test step that have yet to be executed. ''' def __init__(self, test: _TestStepWithPlaceholders, runtime_config_variable_storage: dict): @@ -443,7 +446,7 @@ def post_process_response(self, response: dict): return result def _skip_post_processing(self, response: dict, result) -> bool: - ''' Should we skip perform post processing. + '''Should we skip perform post processing. Currently we only skip post processing if the test step indicates that sent test step invokation was expected to be optionally supported. We confirm that it is optional From 005cc7ebd9fa4dd569ed170a17952cca965410e5 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Tue, 13 Dec 2022 19:02:31 +0000 Subject: [PATCH 18/23] Attempt at creating a python package --- scripts/tests/yamltests/BUILD.gn | 43 +++++++++++++++++++++++++++++ scripts/tests/yamltests/__init__.py | 0 scripts/tests/yamltests/setup.py | 28 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 scripts/tests/yamltests/BUILD.gn create mode 100644 scripts/tests/yamltests/__init__.py create mode 100644 scripts/tests/yamltests/setup.py diff --git a/scripts/tests/yamltests/BUILD.gn b/scripts/tests/yamltests/BUILD.gn new file mode 100644 index 00000000000000..4655c0063415f2 --- /dev/null +++ b/scripts/tests/yamltests/BUILD.gn @@ -0,0 +1,43 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python.gni") + +pw_python_package("yamltests") { + setup = [ "setup.py" ] + + sources = [ + "__init__.py", + "constraints.py", + "definitions.py", + "fixes.py", + "parser.py", + ] + + python_deps = [ + "${chip_root}/scripts/idl", + ] + + tests = [ + "test_spec_definitions.py", + ] + + # TODO: at a future time consider enabling all (* or missing) here to get + # pylint checking these files + static_analysis = [] +} diff --git a/scripts/tests/yamltests/__init__.py b/scripts/tests/yamltests/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/scripts/tests/yamltests/setup.py b/scripts/tests/yamltests/setup.py new file mode 100644 index 00000000000000..2bcdfbe8c61d46 --- /dev/null +++ b/scripts/tests/yamltests/setup.py @@ -0,0 +1,28 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""The yamltest package.""" + +import setuptools # type: ignore + +setuptools.setup( + name='yamltests', + version='0.0.1', + author='Project CHIP Authors', + description='Parse matter yaml test files', + packages=setuptools.find_packages(), + package_data={'yamltest': ['py.typed']}, + zip_safe=False, +) From 6151607c87d7734ff05926da6c5b3e25e7074a37 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Wed, 14 Dec 2022 14:31:11 +0000 Subject: [PATCH 19/23] Make Restyle Happy --- scripts/tests/yamltests/BUILD.gn | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/tests/yamltests/BUILD.gn b/scripts/tests/yamltests/BUILD.gn index 4655c0063415f2..2f9a49c0744181 100644 --- a/scripts/tests/yamltests/BUILD.gn +++ b/scripts/tests/yamltests/BUILD.gn @@ -29,13 +29,9 @@ pw_python_package("yamltests") { "parser.py", ] - python_deps = [ - "${chip_root}/scripts/idl", - ] + python_deps = [ "${chip_root}/scripts/idl" ] - tests = [ - "test_spec_definitions.py", - ] + tests = [ "test_spec_definitions.py" ] # TODO: at a future time consider enabling all (* or missing) here to get # pylint checking these files From 08a1677ac1b43d6073d37191ce15ef5a68760404 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Wed, 14 Dec 2022 18:19:15 +0000 Subject: [PATCH 20/23] Clean up yaml parser unit test to be self contained --- scripts/tests/test_yaml_parser.py | 98 +++++++++++++++---------------- 1 file changed, 47 insertions(+), 51 deletions(-) diff --git a/scripts/tests/test_yaml_parser.py b/scripts/tests/test_yaml_parser.py index 787e1a15e21e76..5179cdcddd308a 100644 --- a/scripts/tests/test_yaml_parser.py +++ b/scripts/tests/test_yaml_parser.py @@ -19,75 +19,71 @@ # to a more appropriate spot. For now, having this file to do some quick checks # is arguably better then no checks at all. -import glob -import os +import io +import tempfile import unittest -import functools -from pathlib import Path from yamltests.definitions import * from yamltests.parser import TestParser -_DEFAULT_MATTER_ROOT = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', '..')) -_YAML_TEST_SUITE_PATH = os.path.abspath( - os.path.join(_DEFAULT_MATTER_ROOT, 'src/app/tests/suites')) +source_struct = ''' + + + + + -_CLUSTER_DEFINITION_DIRECTORY = os.path.abspath( - os.path.join(_DEFAULT_MATTER_ROOT, 'src/app/zap-templates/zcl/data-model')) + + Test + 0x1234 + + +''' -def sort_with_global_attribute_first(a, b): - if a.endswith('global-attributes.xml'): - return -1 - elif b.endswith('global-attributes.xml'): - return 1 - elif a > b: - return 1 - elif a == b: - return 0 - elif a < b: - return -1 +simple_test_yaml = ''' +name: Test Cluster Tests +config: + nodeId: 0x12344321 + cluster: "Test" + endpoint: 1 -class TestYamlParser(unittest.TestCase): - def setUp(self): - # TODO we should not be reliant on an external YAML file. Possible that we should have - # either a local .yaml testfile, of the file contents should exist in this file where we - # write out the yaml file to temp directory for our testing use. This approach was taken - # since this test is better than no test. - yaml_test_suite_path = Path(_YAML_TEST_SUITE_PATH) - if not yaml_test_suite_path.exists(): - raise FileNotFoundError(f'Expected directory {_YAML_TEST_SUITE_PATH} to exist') - yaml_test_filename = 'TestCluster.yaml' - path_to_test = None - for path in yaml_test_suite_path.rglob(yaml_test_filename): - if not path.is_file(): - continue - if path.name != yaml_test_filename: - continue - path_to_test = str(path) - break - if path_to_test is None: - raise FileNotFoundError(f'Could not file {yaml_test_filename} in directory {_YAML_TEST_SUITE_PATH}') - pics_file = None +tests: + - label: "Send Test Command" + command: "test" + + - label: "Send Test Not Handled Command" + command: "testNotHandled" + response: + error: INVALID_COMMAND - # TODO Again we should not be reliant on extneral XML files. But some test (even brittal) - # are better than no tests. + - label: "Send Test Specific Command" + command: "testSpecific" + response: + values: + - name: "returnValue" + value: 7 +''' - filenames = glob.glob(_CLUSTER_DEFINITION_DIRECTORY + '/*/*.xml', recursive=False) - filenames.sort(key=functools.cmp_to_key(sort_with_global_attribute_first)) - sources = [ParseSource(source=name) for name in filenames] - specifications = SpecDefinitions(sources) - self._yaml_parser = TestParser(path_to_test, pics_file, specifications) +class TestYamlParserNew(unittest.TestCase): + def setUp(self): + self._definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_struct), name='source_struct')]) + self._temp_file = tempfile.NamedTemporaryFile(suffix='.yaml') + with open(self._temp_file.name, 'w') as f: + f.writelines(simple_test_yaml) + pics_file = None + self._yaml_parser = TestParser(self._temp_file.name, pics_file, self._definitions) - def test_able_to_iterate_over_all_tests(self): + def test_foobar(self): # self._yaml_parser.tests implements `__next__`, which does value substitution. We are # simply ensure there is no exceptions raise. + count = 0 for idx, test_step in enumerate(self._yaml_parser.tests): + count += 1 pass - self.assertTrue(True) + self.assertEqual(count, 3) def main(): From 7b2100dacde9534236f7caa7c1d349e61f1394a6 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Wed, 14 Dec 2022 18:21:07 +0000 Subject: [PATCH 21/23] Fix test name --- scripts/tests/test_yaml_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_yaml_parser.py b/scripts/tests/test_yaml_parser.py index 5179cdcddd308a..127c78c0a6d79a 100644 --- a/scripts/tests/test_yaml_parser.py +++ b/scripts/tests/test_yaml_parser.py @@ -67,7 +67,7 @@ ''' -class TestYamlParserNew(unittest.TestCase): +class TestYamlParser(unittest.TestCase): def setUp(self): self._definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_struct), name='source_struct')]) self._temp_file = tempfile.NamedTemporaryFile(suffix='.yaml') @@ -76,7 +76,7 @@ def setUp(self): pics_file = None self._yaml_parser = TestParser(self._temp_file.name, pics_file, self._definitions) - def test_foobar(self): + def test_able_to_iterate_over_all_parsed_tests(self): # self._yaml_parser.tests implements `__next__`, which does value substitution. We are # simply ensure there is no exceptions raise. count = 0 From 8573cfd50f87520ebff7bca4442afbbcb5680443 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Wed, 14 Dec 2022 18:23:43 +0000 Subject: [PATCH 22/23] Fix test name --- scripts/tests/test_yaml_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_yaml_parser.py b/scripts/tests/test_yaml_parser.py index 127c78c0a6d79a..c6a73da4f8eed9 100644 --- a/scripts/tests/test_yaml_parser.py +++ b/scripts/tests/test_yaml_parser.py @@ -26,7 +26,7 @@ from yamltests.definitions import * from yamltests.parser import TestParser -source_struct = ''' +simple_test_description = ''' @@ -69,7 +69,8 @@ class TestYamlParser(unittest.TestCase): def setUp(self): - self._definitions = SpecDefinitions([ParseSource(source=io.StringIO(source_struct), name='source_struct')]) + self._definitions = SpecDefinitions( + [ParseSource(source=io.StringIO(simple_test_description), name='simple_test_description')]) self._temp_file = tempfile.NamedTemporaryFile(suffix='.yaml') with open(self._temp_file.name, 'w') as f: f.writelines(simple_test_yaml) From 0036de8485dbbc83ca84648a0608c8fc87a95f51 Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Thu, 15 Dec 2022 14:28:38 +0000 Subject: [PATCH 23/23] Address PR comments --- scripts/tests/yamltests/definitions.py | 31 +++++++++++++------------- scripts/tests/yamltests/parser.py | 28 +++++++++-------------- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/scripts/tests/yamltests/definitions.py b/scripts/tests/yamltests/definitions.py index 0bd70de5f2a769..006704993d7bab 100644 --- a/scripts/tests/yamltests/definitions.py +++ b/scripts/tests/yamltests/definitions.py @@ -40,23 +40,24 @@ class _ItemType(enum.Enum): class SpecDefinitions: - __clusters_by_id: dict[int, Cluster] = {} - __commands_by_id: dict[int, dict[int, Command]] = {} - __responses_by_id: dict[int, dict[int, Struct]] = {} - __attributes_by_id: dict[int, dict[int, Attribute]] = {} - __events_by_id: dict[int, dict[int, Event]] = {} - - __clusters_by_name: dict[str, int] = {} - __commands_by_name: dict[str, int] = {} - __responses_by_name: dict[str, int] = {} - __attributes_by_name: dict[str, int] = {} - __events_by_name: dict[str, int] = {} - - __bitmaps_by_name: dict[str, dict[str, Bitmap]] = {} - __enums_by_name: dict[str, dict[str, Enum]] = {} - __structs_by_name: dict[str, dict[str, Struct]] = {} def __init__(self, sources: List[ParseSource]): + self.__clusters_by_id: dict[int, Cluster] = {} + self.__commands_by_id: dict[int, dict[int, Command]] = {} + self.__responses_by_id: dict[int, dict[int, Struct]] = {} + self.__attributes_by_id: dict[int, dict[int, Attribute]] = {} + self.__events_by_id: dict[int, dict[int, Event]] = {} + + self.__clusters_by_name: dict[str, int] = {} + self.__commands_by_name: dict[str, int] = {} + self.__responses_by_name: dict[str, int] = {} + self.__attributes_by_name: dict[str, int] = {} + self.__events_by_name: dict[str, int] = {} + + self.__bitmaps_by_name: dict[str, dict[str, Bitmap]] = {} + self.__enums_by_name: dict[str, dict[str, Enum]] = {} + self.__structs_by_name: dict[str, dict[str, Struct]] = {} + idl = ParseXmls(sources) for cluster in idl.clusters: diff --git a/scripts/tests/yamltests/parser.py b/scripts/tests/yamltests/parser.py index 1f8b8a63321cff..329d17cf124856 100644 --- a/scripts/tests/yamltests/parser.py +++ b/scripts/tests/yamltests/parser.py @@ -79,14 +79,14 @@ class PostProcessCheckStatus(Enum): - '''Inidcates the post processing check step status.''' + '''Indicates the post processing check step status.''' SUCCESS = 'success', WARNING = 'warning', ERROR = 'error' class PostProcessCheckType(Enum): - '''Inidcates the post processing check step type.''' + '''Indicates the post processing check step type.''' IM_STATUS = 'IMStatus', CLUSTER_STATUS = 'ClusterStatus', RESPONSE_VALIDATION = 'Response', @@ -181,10 +181,6 @@ class _TestStepWithPlaceholders: is only known after an earlier test step's has executed and the result successfully post processed. ''' - is_enabled = True - is_command = False - is_attribute = False - is_event = False def __init__(self, test: dict, config: dict, definitions): # Disabled tests are not parsed in order to allow the test to be added to the test @@ -366,10 +362,6 @@ def __init__(self, test: _TestStepWithPlaceholders, runtime_config_variable_stor def is_enabled(self): return self._test.is_enabled - @property - def is_command(self): - return self._test.is_command - @property def is_attribute(self): return self._test.is_attribute @@ -655,10 +647,15 @@ def _config_variable_substitution(self, value): class YamlTests: - _tests: list[_TestStepWithPlaceholders] - _index: 0 + '''Parses YAML tests and becomes an iterator to provide 'TestStep's - count: 0 + The provided TestStep is expected to be used by a runner/adapter to run the test step and + provide the response from the device to the TestStep object. + + Currently this is a one time use object. Eventually this should be refactored to take a + runner/adapter as an argument and run through all test steps and should be reusable for + multiple runs. + ''' def __init__(self, parsing_config_variable_storage: dict, definitions, tests: dict): self._parsing_config_variable_storage = parsing_config_variable_storage @@ -690,11 +687,6 @@ def __next__(self) -> TestStep: class TestParser: - name: None - PICS: None - tests: None - _parsing_config_variable_storage: dict = {} - def __init__(self, test_file, pics_file, definitions): # TODO Needs supports for PICS file with open(test_file) as f: