From 3e1cc304567b7a4dc0c6e81f33a41628fd384582 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 20 Oct 2023 04:24:47 -0400 Subject: [PATCH] Create a codegen that converts idl back into .matter formats (#29867) * Start creating a IDL codegen so we can self-test parsed output * Start with listing clusters * Enum listing * A lot more things supported * Attribute rendering * Support for string and octet string sizes * Timed command support * Restyle * Add descriptions to clusters * Attempt to fix up alignment of things * Alignment looks slightly better * Better command separation * Align comments * Align and output descriptions including clusters * More work regarding loop structures * Apply hex formatting to bitmaps. output now seems identical except one whitespace change * Identical output for now * Support API maturity. Notice that doccomments are lost on maturity :( * Fix doxygen parsing for api maturity at the cluster level * Restyle * Support endpoints, although that is not 1:1 as hex encoding and ordering for events is lost * Restyle * Add todo note that default value does not string escaping * Default rendering and add to files * More updates on file dependencies * Unit test IDL generator * Add the IDL unit test as a standard unit test * Update for python compatibility * Fix unit testing of builds when GSDK root is defined * Added a readme file * Restyle * Make xml parser use the idl codegen * Restyle * look to fix misspell warnings * Undo repo update * Fix linter errors --------- Co-authored-by: Andrei Litvin --- scripts/build/test.py | 1 + ...un_efr32-brd4161a-light-rpc-no-version.txt | 2 +- scripts/py_matter_idl/BUILD.gn | 1 + scripts/py_matter_idl/files.gni | 2 + .../matter_idl/generators/idl/MatterIdl.jinja | 150 +++++++++++++ .../matter_idl/generators/idl/README.md | 32 +++ .../matter_idl/generators/idl/__init__.py | 200 ++++++++++++++++++ .../matter_idl/generators/registry.py | 5 + .../matter_idl/matter_idl_parser.py | 8 + .../matter_idl/test_idl_generator.py | 115 ++++++++++ .../py_matter_idl/matter_idl/xml_parser.py | 26 ++- scripts/py_matter_idl/setup.cfg | 1 + 12 files changed, 536 insertions(+), 7 deletions(-) create mode 100644 scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja create mode 100644 scripts/py_matter_idl/matter_idl/generators/idl/README.md create mode 100644 scripts/py_matter_idl/matter_idl/generators/idl/__init__.py create mode 100755 scripts/py_matter_idl/matter_idl/test_idl_generator.py diff --git a/scripts/build/test.py b/scripts/build/test.py index c89a4d98109357..f684ace074d5ca 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -48,6 +48,7 @@ def build_actual_output(root: str, out: str, args: List[str]) -> List[str]: 'IMX_SDK_ROOT': 'IMX_SDK_ROOT', 'TI_SYSCONFIG_ROOT': 'TEST_TI_SYSCONFIG_ROOT', 'JAVA_PATH': 'TEST_JAVA_PATH', + 'GSDK_ROOT': 'TEST_GSDK_ROOT', }) retval = subprocess.run([ diff --git a/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt b/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt index 309cdd49b9f256..b96ccbc89ea97e 100644 --- a/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt +++ b/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt @@ -2,7 +2,7 @@ cd "{root}" # Generating efr32-brd4161a-light-rpc-no-version -gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/lighting-app/silabs '--args=silabs_board="BRD4161A" is_debug=false import("//with_pw_rpc.gni")' {out}/efr32-brd4161a-light-rpc-no-version +gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/lighting-app/silabs '--args=silabs_board="BRD4161A" is_debug=false import("//with_pw_rpc.gni") efr32_sdk_root="TEST_GSDK_ROOT" openthread_root="TEST_GSDK_ROOT/util/third_party/openthread"' {out}/efr32-brd4161a-light-rpc-no-version # Building efr32-brd4161a-light-rpc-no-version ninja -C {out}/efr32-brd4161a-light-rpc-no-version diff --git a/scripts/py_matter_idl/BUILD.gn b/scripts/py_matter_idl/BUILD.gn index 81f5be3be554b4..48fc68aea40268 100644 --- a/scripts/py_matter_idl/BUILD.gn +++ b/scripts/py_matter_idl/BUILD.gn @@ -62,6 +62,7 @@ pw_python_package("matter_idl") { "matter_idl/test_backwards_compatibility.py", "matter_idl/test_matter_idl_parser.py", "matter_idl/test_generators.py", + "matter_idl/test_idl_generator.py", "matter_idl/test_xml_parser.py", ] diff --git a/scripts/py_matter_idl/files.gni b/scripts/py_matter_idl/files.gni index 84680a8863d42f..cb54b253ef0615 100644 --- a/scripts/py_matter_idl/files.gni +++ b/scripts/py_matter_idl/files.gni @@ -7,6 +7,7 @@ matter_idl_generator_templates = [ "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/PluginApplicationCallbacksHeader.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_cpp.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_h.jinja", + "${chip_root}/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/CHIPCallbackTypes.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersCpp.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersRead.jinja", @@ -27,6 +28,7 @@ matter_idl_generator_sources = [ "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/filters.py", + "${chip_root}/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/registry.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/types.py", diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja new file mode 100644 index 00000000000000..91e978a44c7b07 --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -0,0 +1,150 @@ +{% macro render_field(field) -%}{# + Macro for the output of a single field entry such as: + + int16u identifyTime = 0; + optional int16u transitionTime = 3; + optional nullable int16u transitionTime = 2; + optional ExtensionFieldSet extensionFieldSets[] = 5; +#} + +{%- if field.qualities %}{{field.qualities | idltxt}} {% endif -%} +{{field.data_type.name}} +{%- if field.data_type.max_length -%} <{{field.data_type.max_length}}> {%- endif -%} +{##} {{field.name}} +{%- if field.is_list %}[]{% endif -%} +{##} = {{field.code}}; +{%- endmacro -%} + +{% macro render_struct(s) -%}{# + Macro for the output of a complete struct +#} + {%- if s.tag %}{{s.tag | idltxt}} {% endif -%} + {% if s.qualities %}{{s.qualities | idltxt}} {% endif -%} + struct {{s.name}} {##} + {%- if s.code is not none %}= {{s.code}} {% endif -%} + { + {% for field in s.fields %} + {{render_field(field)}} + {% endfor %} + } +{%- endmacro -%} + + +// This IDL was auto-generated from a parsed data structure + +{% for cluster in idl.clusters %} +{% if cluster.description %}/** {{cluster.description}} */ +{% endif %} +{{cluster.api_maturity | idltxt}}{{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { + {%- for enum in cluster.enums %} + + enum {{enum.name}} : {{ enum.base_type}} { + {% for entry in enum.entries %} + {{entry.name}} = {{entry.code}}; + {% endfor %} + } + {% endfor %} + + {%- for bitmap in cluster.bitmaps %} + + bitmap {{bitmap.name}} : {{ bitmap.base_type}} { + {% for entry in bitmap.entries %} + {{entry.name}} = 0x{{"%X" | format(entry.code)}}; + {% endfor %} + } + {% endfor %} + + {%- for s in cluster.structs | rejectattr("tag") %} + {% if loop.first %} + + {% endif %} + {{render_struct(s)}} + {% if not loop.last %} + + {% endif %} + {% endfor %} + + {%- for e in cluster.events %} + {% if loop.first %} + + {% endif %} + {##} {##}{% if e.qualities %}{{e.qualities | idltxt}} {% endif -%} + {{e.priority | idltxt}} event {{e | event_access}}{{e.name}} = {{e.code}} { + {% for field in e.fields %} + {{render_field(field)}} + {% endfor %} + } + {% if not loop.last %} + + {% endif %} + {% endfor %} + + {%- for a in cluster.attributes %} + {% if loop.first %} + + {% endif %} + {{a.qualities | idltxt}}attribute {{a | attribute_access}}{{render_field(a.definition)}} + {% endfor %} + + {%- for s in cluster.structs | selectattr("tag") %} + + {{render_struct(s)}} + {% endfor %} + + {%- for c in cluster.commands %} + {% if loop.first %} + + {% endif %} + {% if c.description %} + /** {{c.description}} */ + {% endif %} + {{c.qualities | idltxt}}command {{c | command_access}}{{c.name}}( + {%- if c.input_param %}{{c.input_param}}{% endif -%} + ): {{c.output_param}} = {{c.code}}; + {% endfor %} +} + +{% endfor %} + +{%- if idl.endpoints %} +{%- for endpoint in idl.endpoints %} +endpoint {{endpoint.number}} { + {% for t in endpoint.device_types %} + device type {{t.name}} = {{t.code}}, version {{t.version}}; + {% endfor%} + + {%-for b in endpoint.client_bindings %} + {% if loop.first %} + + {% endif %} + binding cluster {{b}}; + {% endfor %} + + {%-for c in endpoint.server_clusters %} + + server cluster {{c.name}} { + {% for e in c.events_emitted %} + emits event {{e}}; + {% if loop.last %} + + {% endif %} + {% endfor %} + {% for a in c.attributes %} + {{"%-8s" | format(a.storage|idltxt) }} attribute {{a.name}} + {%- if a.default is not none %} default = {{a.default|render_default}} {%- endif %}; + {% endfor %} + {% for cmd in c.commands %} + {% if loop.first %} + + {% endif %} + handle command {{cmd.name}}; + {% endfor %} + } + {% endfor %} + +} +{% if not loop.last %} + +{% endif %} +{% endfor %} +{% endif %} diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/README.md b/scripts/py_matter_idl/matter_idl/generators/idl/README.md new file mode 100644 index 00000000000000..81e4b92c05e535 --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/generators/idl/README.md @@ -0,0 +1,32 @@ +## Generator description + +Generates a structured `Idl` data type into a human-readable text format +(`.matter` file). + +It is useful for tools that ingest non-idl data but convert into idl data (e.g. +`zapxml` or CSA data model XML data.) + +### Usage + +A no-op usage can be: + +``` +./scripts/codegen.py -g idl --output-dir out/idlgen examples/all-clusters-app/all-clusters-common/all-clusters-app.matter +``` + +which would re-generate the entire `all-clusters-app.matter` into +`out/idlgen/idl.matter` + +This generation is useful for testing/validating that both parsing and +generation works. Actual usage of this generator would be inside XML tools. + +### Within XML parsing + +A XML parser will use this code generator to output a human readable view of the +parsed data: + +``` +./scripts/py_matter_idl/matter_idl/xml_parser.py \ + ./src/app/zap-templates/zcl/data-model/chip/onoff-cluster.xml \ + ./src/app/zap-templates/zcl/data-model/chip/global-attributes.xm +``` diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py new file mode 100644 index 00000000000000..f53e35a7aba8e0 --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -0,0 +1,200 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Union + +from matter_idl.generators import CodeGenerator, GeneratorStorage +from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, AttributeStorage, ClusterSide, + Command, CommandQuality, Event, EventPriority, EventQuality, FieldQuality, Idl, + StructQuality, StructTag) + + +def human_text_string(value: Union[ClusterSide, StructTag, StructQuality, EventPriority, EventQuality, AccessPrivilege, AttributeQuality, CommandQuality, ApiMaturity, AttributeStorage]) -> str: + if type(value) is ClusterSide: + if value == ClusterSide.CLIENT: + return "client" + if value == ClusterSide.SERVER: + return "server" + elif type(value) is StructTag: + if value == StructTag.REQUEST: + return "request" + if value == StructTag.RESPONSE: + return "response" + elif type(value) is FieldQuality: + result = "" + if FieldQuality.OPTIONAL in value: + result += "optional " + if FieldQuality.NULLABLE in value: + result += "nullable " + if FieldQuality.FABRIC_SENSITIVE in value: + result += "fabric_sensitive " + return result.strip() + elif type(value) is StructQuality: + result = "" + if value == StructQuality.FABRIC_SCOPED: + result += "fabric_scoped " + return result.strip() + elif type(value) is EventPriority: + if value == EventPriority.DEBUG: + return "debug" + if value == EventPriority.INFO: + return "info" + if value == EventPriority.CRITICAL: + return "critical" + elif type(value) is EventQuality: + result = "" + if EventQuality.FABRIC_SENSITIVE in value: + result += "fabric_sensitive " + return result.strip() + elif type(value) is AccessPrivilege: + if value == AccessPrivilege.VIEW: + return "view" + if value == AccessPrivilege.OPERATE: + return "operate" + if value == AccessPrivilege.MANAGE: + return "manage" + if value == AccessPrivilege.ADMINISTER: + return "administer" + elif type(value) is AttributeQuality: + result = "" + if AttributeQuality.TIMED_WRITE in value: + result += "timedwrite " + if AttributeQuality.WRITABLE not in value: + result += "readonly " + if AttributeQuality.NOSUBSCRIBE in value: + result += "nosubscribe " + return result + elif type(value) is CommandQuality: + result = "" + if CommandQuality.TIMED_INVOKE in value: + result += "timed " + if CommandQuality.FABRIC_SCOPED in value: + result += "fabric " + return result + elif type(value) is ApiMaturity: + if value == ApiMaturity.STABLE: + return "" + if value == ApiMaturity.PROVISIONAL: + return "provisional " + if value == ApiMaturity.INTERNAL: + return "internal " + if value == ApiMaturity.DEPRECATED: + return "deprecated " + elif type(value) is AttributeStorage: + if value == AttributeStorage.RAM: + return "ram" + if value == AttributeStorage.PERSIST: + return "persist" + if value == AttributeStorage.CALLBACK: + return "callback" + + # wrong value in general + return "Unknown/unsupported: %r" % value + + +def event_access_string(e: Event) -> str: + """Generates the access string required for an event. If string is non-empty it will + include a trailing space + """ + result = "" + if e.readacl != AccessPrivilege.VIEW: + result += "read: " + human_text_string(e.readacl) + + if not result: + return "" + return f"access({result}) " + + +def command_access_string(c: Command) -> str: + """Generates the access string required for a command. If string is non-empty it will + include a trailing space + """ + result = "" + if c.invokeacl != AccessPrivilege.OPERATE: + result += "invoke: " + human_text_string(c.invokeacl) + + if not result: + return "" + return f"access({result}) " + + +def attribute_access_string(a: Attribute) -> str: + """Generates the access string required for a struct. If string is non-empty it will + include a trailing space + """ + result = [] + + if a.readacl != AccessPrivilege.VIEW: + result.append("read: " + human_text_string(a.readacl)) + + if a.writeacl != AccessPrivilege.OPERATE: + result.append("write: " + human_text_string(a.writeacl)) + + if not result: + return "" + + return f"access({', '.join(result)}) " + + +def render_default(value: Union[str, int, bool]) -> str: + """ + Renders a idl-style default. + + Generally quotes strings and handles bools + """ + if type(value) is str: + # TODO: technically this should support escaping for quotes + # however currently we never needed this. Escaping can be + # added once we use this info + return f'"{value}"' + elif type(value) is bool: + if value: + return "true" + else: + return "false" + return str(value) + + +class IdlGenerator(CodeGenerator): + """ + Generation .matter idl files for a given IDL + """ + + def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): + super().__init__(storage, idl, fs_loader_searchpath=os.path.dirname(__file__)) + + self.jinja_env.filters['idltxt'] = human_text_string + self.jinja_env.filters['event_access'] = event_access_string + self.jinja_env.filters['command_access'] = command_access_string + self.jinja_env.filters['attribute_access'] = attribute_access_string + self.jinja_env.filters['render_default'] = render_default + + # Easier whitespace management + self.jinja_env.trim_blocks = True + self.jinja_env.lstrip_blocks = True + + def internal_render_all(self): + """ + Renders the output. + """ + + # Header containing a macro to initialize all cluster plugins + self.internal_render_one_output( + template_path="MatterIdl.jinja", + output_file_name="idl.matter", + vars={ + 'idl': self.idl + } + ) diff --git a/scripts/py_matter_idl/matter_idl/generators/registry.py b/scripts/py_matter_idl/matter_idl/generators/registry.py index e3c8e1ed32397e..b02b9bbcf12f0b 100644 --- a/scripts/py_matter_idl/matter_idl/generators/registry.py +++ b/scripts/py_matter_idl/matter_idl/generators/registry.py @@ -17,6 +17,7 @@ from matter_idl.generators.cpp.application import CppApplicationGenerator from matter_idl.generators.cpp.tlvmeta import TLVMetaDataGenerator +from matter_idl.generators.idl import IdlGenerator from matter_idl.generators.java import JavaClassGenerator, JavaJNIGenerator @@ -30,6 +31,7 @@ class CodeGenerator(enum.Enum): JAVA_CLASS = enum.auto() CPP_APPLICATION = enum.auto() CPP_TLVMETA = enum.auto() + IDL = enum.auto() CUSTOM = enum.auto() def Create(self, *args, **kargs): @@ -41,6 +43,8 @@ def Create(self, *args, **kargs): return CppApplicationGenerator(*args, **kargs) elif self == CodeGenerator.CPP_TLVMETA: return TLVMetaDataGenerator(*args, **kargs) + elif self == CodeGenerator.IDL: + return IdlGenerator(*args, **kargs) elif self == CodeGenerator.CUSTOM: # Use a package naming convention to find the custom generator: # ./matter_idl_plugin/__init__.py defines a subclass of CodeGenerator named CustomGenerator. @@ -70,5 +74,6 @@ def FromString(name): 'java-class': CodeGenerator.JAVA_CLASS, 'cpp-app': CodeGenerator.CPP_APPLICATION, 'cpp-tlvmeta': CodeGenerator.CPP_TLVMETA, + 'idl': CodeGenerator.IDL, 'custom': CodeGenerator.CUSTOM, } diff --git a/scripts/py_matter_idl/matter_idl/matter_idl_parser.py b/scripts/py_matter_idl/matter_idl/matter_idl_parser.py index 0d1bfb8d869e49..711e2535887b5e 100755 --- a/scripts/py_matter_idl/matter_idl/matter_idl_parser.py +++ b/scripts/py_matter_idl/matter_idl/matter_idl_parser.py @@ -46,6 +46,14 @@ def appply_to_idl(self, idl: Idl, content: str): while content[actual_pos] in ' \t\n\r': actual_pos += 1 + # Allow to skip api maturity flags + for maturity in ["provisional", "internal", "stable", "deprecated"]: + if content[actual_pos:].startswith(maturity): + actual_pos += len(maturity) + + while content[actual_pos] in ' \t\n\r': + actual_pos += 1 + # A doc comment will apply to any supported element assuming it immediately # preceeds id (skipping whitespace) for item in self.supported_types(idl): diff --git a/scripts/py_matter_idl/matter_idl/test_idl_generator.py b/scripts/py_matter_idl/matter_idl/test_idl_generator.py new file mode 100755 index 00000000000000..ea1347fb0e7d4c --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/test_idl_generator.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest +from typing import Optional + +try: + from matter_idl.matter_idl_parser import CreateParser +except ImportError: + + sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..'))) + from matter_idl.matter_idl_parser import CreateParser + +from matter_idl.generators import GeneratorStorage +from matter_idl.generators.idl import IdlGenerator +from matter_idl.matter_idl_types import Idl + + +class TestCaseStorage(GeneratorStorage): + def __init__(self): + super().__init__() + self.content: Optional[str] = None + + def get_existing_data(self, relative_path: str): + # Force re-generation each time + return None + + def write_new_data(self, relative_path: str, content: str): + if self.content: + raise Exception("Unexpected extra data: single file generation expected") + self.content = content + + +def ReadMatterIdl(repo_path: str) -> Idl: + path = os.path.join(os.path.dirname(__file__), "../../..", repo_path) + with open(path, "rt") as stream: + return stream.read() + + +def ParseMatterIdl(repo_path: str, skip_meta: bool) -> Idl: + return CreateParser(skip_meta=skip_meta).parse(ReadMatterIdl(repo_path)) + + +def RenderAsIdlTxt(idl: Idl) -> str: + storage = TestCaseStorage() + IdlGenerator(storage=storage, idl=idl).render(dry_run=False) + return storage.content + + +def SkipLeadingComments(txt: str) -> str: + """Skips leading lines starting with // in a file. """ + lines = txt.split("\n") + idx = 0 + while lines[idx].startswith("//") or not lines[idx]: + idx = idx + 1 + return "\n".join(lines[idx:]) + + +class TestIdlRendering(unittest.TestCase): + def test_client_clusters(self): + # IDL renderer was updated to have IDENTICAL output for client side + # cluster rendering, so this diff will be verbatim + # + # Comparison made text-mode so that meta-data is read and doc-comments are + # available + + path = "src/controller/data_model/controller-clusters.matter" + + # Files MUST be identical except the header comments which are different + original = SkipLeadingComments(ReadMatterIdl(path)) + generated = SkipLeadingComments(RenderAsIdlTxt(ParseMatterIdl(path, skip_meta=False))) + + self.assertEqual(original, generated) + + def test_app_rendering(self): + # When endpoints are involved, default value formatting is lost + # (e.g. "0x0000" becomes "0") and ordering of emitted events is not preserved + # because the events are a + + # as such, this test validates that parsing + generating + re-parsing results + # in the same data being parsed + test_paths = [ + "examples/lock-app/lock-common/lock-app.matter", + "examples/lighting-app/lighting-common/lighting-app.matter", + "examples/all-clusters-app/all-clusters-common/all-clusters-app.matter", + "examples/thermostat/thermostat-common/thermostat.matter", + ] + + for path in test_paths: + idl = ParseMatterIdl(path, skip_meta=True) + txt = RenderAsIdlTxt(idl) + idl2 = CreateParser(skip_meta=True).parse(txt) + + # checks that data types and content is the same + self.assertEqual(idl, idl2) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/py_matter_idl/matter_idl/xml_parser.py b/scripts/py_matter_idl/matter_idl/xml_parser.py index 5a07490e9c3b68..d20bdb9dda67ec 100755 --- a/scripts/py_matter_idl/matter_idl/xml_parser.py +++ b/scripts/py_matter_idl/matter_idl/xml_parser.py @@ -16,6 +16,7 @@ import logging import os +from typing import Optional try: from matter_idl.zapxml import ParseSource, ParseXmls @@ -28,11 +29,23 @@ if __name__ == '__main__': - # This Parser is generally not intended to be run as a stand-alone binary. - # The ability to run is for debug and to print out the parsed AST. - import pprint - import click + from matter_idl.generators import GeneratorStorage + from matter_idl.generators.idl import IdlGenerator + + class InMemoryStorage(GeneratorStorage): + def __init__(self): + super().__init__() + self.content: Optional[str] = None + + def get_existing_data(self, relative_path: str): + # Force re-generation each time + return None + + def write_new_data(self, relative_path: str, content: str): + if self.content: + raise Exception("Unexpected extra data: single file generation expected") + self.content = content # Supported log levels, mapping string values required for argument # parsing into logging constants @@ -70,7 +83,8 @@ def main(log_level, no_print, filenames): logging.info("Parse completed") if not no_print: - print("Data:") - pprint.pp(data) + storage = InMemoryStorage() + IdlGenerator(storage=storage, idl=data).render(dry_run=False) + print(storage.content) main(auto_envvar_prefix='CHIP') diff --git a/scripts/py_matter_idl/setup.cfg b/scripts/py_matter_idl/setup.cfg index 57f93c3dd9aa82..714b42a46e614e 100644 --- a/scripts/py_matter_idl/setup.cfg +++ b/scripts/py_matter_idl/setup.cfg @@ -34,6 +34,7 @@ matter_idl = generators/cpp/application/PluginApplicationCallbacksHeader.jinja generators/cpp/tlvmeta/TLVMetaData_cpp.jinja generators/cpp/tlvmeta/TLVMetaData_h.jinja + generators/idl/MatterIdl.jinja generators/java/CHIPCallbackTypes.jinja generators/java/ChipClustersCpp.jinja generators/java/ChipClustersRead.jinja