From b49b8456d371babd14acd9ebffb0f701fddef281 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani <152916324+vatsalghelani-csa@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:57:03 -0500 Subject: [PATCH] Updated spec_parsing.py to now use the data model directory from python package (#36596) * Added modified spec_parsing.py to now use the data model package * Restyled by autopep8 * Restyled by isort * Fixing the cosmetic changes * Use chip.testing as a module to extract, work via PosixPath directory * Restyled by autopep8 * Restyled by isort * Fixed correct directory search * Solving the wrapping for try-catch according to comments * Restyled by autopep8 * Added fixes for both a pre-built location or a full path * Fix comment * Restyled by autopep8 * Adding importlib.resources capability * Restyled by autopep8 * Restyled by isort * Fixed _spec_ path error * Fixed to use module as string * Removed unused import * Added fixes for path issues * Fixing the xml not found error * Restyled by autopep8 * Fixed code lint error * Fixed code lint error tree * Fixed importlib errors * Fixed code lint * Fixed errors * Restyled by autopep8 * Some type updates and iteration logic updates to be consistent * Remove unused method * Fix logic to match existing usage: we need clusters to be part of the passed in path if applicable * Restyled by autopep8 * Restyled by isort * remove unused import * Cleanup some odd comments * Another update to avoid using globs * Fix up types and return * Remove unused import * Another dep cleanup * Remove one test step: unclear about the value of throwing a specparse exception on invalid input type * Remove unused import * update logic to throw specparsing when no XMLs found ... this preserves previous logic somewhat * Make data model directory consistent with cluster logic * Comments update * Added warning levels for checking xml --------- Co-authored-by: Restyled.io Co-authored-by: Andrei Litvin Co-authored-by: Andrei Litvin --- src/python_testing/TestSpecParsingSupport.py | 5 +- .../chip/testing/spec_parsing.py | 143 +++++++++++------- 2 files changed, 91 insertions(+), 57 deletions(-) diff --git a/src/python_testing/TestSpecParsingSupport.py b/src/python_testing/TestSpecParsingSupport.py index b4c908c232fa94..7ab5847a276883 100644 --- a/src/python_testing/TestSpecParsingSupport.py +++ b/src/python_testing/TestSpecParsingSupport.py @@ -21,7 +21,7 @@ import jinja2 from chip.testing.global_attribute_ids import GlobalAttributeIds from chip.testing.matter_testing import MatterBaseTest, ProblemNotice, default_matter_test_main -from chip.testing.spec_parsing import (ClusterParser, DataModelLevel, PrebuiltDataModelDirectory, SpecParsingException, XmlCluster, +from chip.testing.spec_parsing import (ClusterParser, DataModelLevel, PrebuiltDataModelDirectory, XmlCluster, add_cluster_data_from_xml, build_xml_clusters, check_clusters_for_unknown_commands, combine_derived_clusters_with_base, get_data_model_directory) from mobly import asserts @@ -276,9 +276,6 @@ def test_build_xml_override(self): asserts.assert_count_equal(string_override_check.keys(), self.spec_xml_clusters.keys(), "Mismatched cluster generation") - with asserts.assert_raises(SpecParsingException): - build_xml_clusters("baddir") - def test_spec_parsing_access(self): strs = [None, 'view', 'operate', 'manage', 'admin'] for read in strs: diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 97a13606eabc45..3607f515d3a9ec 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -15,15 +15,16 @@ # limitations under the License. # -import glob +import importlib +import importlib.resources as pkg_resources import logging -import os import typing import xml.etree.ElementTree as ElementTree from copy import deepcopy from dataclasses import dataclass from enum import Enum, auto -from typing import Callable, Optional +from importlib.abc import Traversable +from typing import Callable, Optional, Union import chip.clusters as Clusters import chip.testing.conformance as conformance_support @@ -512,56 +513,83 @@ class PrebuiltDataModelDirectory(Enum): k1_4 = auto() kMaster = auto() - -class DataModelLevel(str, Enum): - kCluster = 'clusters' - kDeviceType = 'device_types' - - -def _get_data_model_root() -> str: - """Attempts to find ${CHIP_ROOT}/data_model or equivalent.""" - - # Since this class is generally in a module, we have to rely on being bootstrapped or - # we use CWD if we cannot - choices = [os.getcwd()] - - if 'PW_PROJECT_ROOT' in os.environ: - choices.insert(0, os.environ['PW_PROJECT_ROOT']) - - for c in choices: - data_model_path = os.path.join(c, 'data_model') - if os.path.exists(os.path.join(data_model_path, 'master', 'scraper_version')): - return data_model_path - raise FileNotFoundError('Cannot find a CHIP_ROOT/data_model path. Tried %r as prefixes.' % choices) - - -def get_data_model_directory(data_model_directory: typing.Union[PrebuiltDataModelDirectory, str], data_model_level: DataModelLevel) -> str: - if data_model_directory == PrebuiltDataModelDirectory.k1_3: - return os.path.join(_get_data_model_root(), '1.3', data_model_level) - elif data_model_directory == PrebuiltDataModelDirectory.k1_4: - return os.path.join(_get_data_model_root(), '1.4', data_model_level) - elif data_model_directory == PrebuiltDataModelDirectory.kMaster: - return os.path.join(_get_data_model_root(), 'master', data_model_level) + @property + def dirname(self): + if self == PrebuiltDataModelDirectory.k1_3: + return "1.3" + if self == PrebuiltDataModelDirectory.k1_4: + return "1.4" + if self == PrebuiltDataModelDirectory.kMaster: + return "master" + raise KeyError("Invalid enum: %r" % self) + + +class DataModelLevel(Enum): + kCluster = auto() + kDeviceType = auto() + + @property + def dirname(self): + if self == DataModelLevel.kCluster: + return "clusters" + if self == DataModelLevel.kDeviceType: + return "device_types" + raise KeyError("Invalid enum: %r" % self) + + +def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable], data_model_level: DataModelLevel = DataModelLevel.kCluster) -> Traversable: + """ + Get the directory of the data model for a specific version and level from the installed package. + + + `data_model_directory` given as a path MUST be of type Traversable (often `pathlib.Path(somepathstring)`). + If `data_model_directory` is given as a Traversable, it is returned directly WITHOUT using the data_model_level at all. + """ + # If it's a prebuilt directory, build the path based on the version and data model level + if isinstance(data_model_directory, PrebuiltDataModelDirectory): + return pkg_resources.files(importlib.import_module('chip.testing')).joinpath( + 'data_model').joinpath(data_model_directory.dirname).joinpath(data_model_level.dirname) else: return data_model_directory -def build_xml_clusters(data_model_directory: typing.Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[uint, XmlCluster], list[ProblemNotice]]: - dir = get_data_model_directory(data_model_directory, DataModelLevel.kCluster) +def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[int, dict], list]: + """ + Build XML clusters from the specified data model directory. + This function supports both pre-built locations and full paths. + + `data_model_directory`` given as a path MUST be of type Traversable (often `pathlib.Path(somepathstring)`). + If data_model_directory is a Travesable, it is assumed to already contain `clusters` (i.e. be a directory + with all XML files in it) + """ clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} ids_by_name: dict[str, int] = {} problems: list[ProblemNotice] = [] - files = glob.glob(f'{dir}/*.xml') - if not files: - raise SpecParsingException(f'No data model files found in specified directory {dir}') - for xml in files: - logging.info(f'Parsing file {xml}') - tree = ElementTree.parse(f'{xml}') - root = tree.getroot() - add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) + top = get_data_model_directory(data_model_directory, DataModelLevel.kCluster) + logging.info("Reading XML clusters from %r", top) + + found_xmls = 0 + for f in top.iterdir(): + if not f.name.endswith('.xml'): + logging.info("Ignoring non-XML file %s", f.name) + continue + + logging.info('Parsing file %s', f.name) + found_xmls += 1 + with f.open("r", encoding="utf8") as file: + root = ElementTree.parse(file).getroot() + add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) + + # For now we assume even a single XML means the directory was probaly OK + # we may increase this later as most our data model directories are larger + # + # Intent here is to make user aware of typos in paths instead of silently having + # empty parsing + if found_xmls < 1: + raise SpecParsingException(f'No data model files found in specified directory {top:!r}') # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. @@ -721,7 +749,7 @@ def combine_attributes(base: dict[uint, XmlAttribute], derived: dict[uint, XmlAt xml_clusters[id] = new -def parse_single_device_type(root: ElementTree.Element) -> tuple[list[ProblemNotice], dict[int, XmlDeviceType]]: +def parse_single_device_type(root: ElementTree.Element) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: problems: list[ProblemNotice] = [] device_types: dict[int, XmlDeviceType] = {} device = root.iter('deviceType') @@ -793,17 +821,26 @@ def parse_single_device_type(root: ElementTree.Element) -> tuple[list[ProblemNot return device_types, problems -def build_xml_device_types(data_model_directory: typing.Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: - dir = get_data_model_directory(data_model_directory, DataModelLevel.kDeviceType) +def build_xml_device_types(data_model_directory: typing.Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: + top = get_data_model_directory(data_model_directory, DataModelLevel.kDeviceType) device_types: dict[int, XmlDeviceType] = {} problems = [] - for xml in glob.glob(f"{dir}/*.xml"): - logging.info(f'Parsing file {xml}') - tree = ElementTree.parse(f'{xml}') - root = tree.getroot() - tmp_device_types, tmp_problems = parse_single_device_type(root) - problems = problems + tmp_problems - device_types.update(tmp_device_types) + + found_xmls = 0 + + for file in top.iterdir(): + if not file.name.endswith('.xml'): + continue + logging.info('Parsing file %r / %s', top, file.name) + found_xmls += 1 + with file.open('r', encoding="utf8") as xml: + root = ElementTree.parse(xml).getroot() + tmp_device_types, tmp_problems = parse_single_device_type(root) + problems = problems + tmp_problems + device_types.update(tmp_device_types) + + if found_xmls < 1: + logging.warning("No XML files found in the specified device type directory: %r", top) if -1 not in device_types.keys(): raise ConformanceException("Base device type not found in device type xml data")