From b7552eaa8172e877d5ba7e885e47c67a0f197681 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 21 Jan 2022 09:11:25 -0500 Subject: [PATCH] Python code capable of parsing matter IDL files (including some unit tests) (#13725) * A IDL parser: Can parse current IDL format (but that may change). Has working unit tests. * one more test * minor comment * make the structs a bit more compact: easier to read * one more tests * more tests, fixed one bug * Add unit test for cluster commands * Unit test for cluster enums * Unit test for cluster events * Rename "structure_member" to field since that seems easier to type and is a good term * Match the newest attribute format for IDLs * Allow test_parsing to be run stand alone and hope that this fix also fixes mac * Rename "parser" to a more specific name: the name parser is used in python and is too generic to attempt a top level import on it * Restyle fixes * Add support for global tag parsing after idl updated in master * Add support for datatype sizes and unit tests * Add test for sized strings in structs as well * Ran restyler --- BUILD.gn | 1 + scripts/idl/BUILD.gn | 35 ++++ scripts/idl/__init__.py | 0 scripts/idl/matter_grammar.lark | 59 ++++++ scripts/idl/matter_idl_parser.py | 240 +++++++++++++++++++++++ scripts/idl/matter_idl_types.py | 135 +++++++++++++ scripts/idl/setup.py | 28 +++ scripts/idl/test_matter_idl_parser.py | 267 ++++++++++++++++++++++++++ scripts/requirements.txt | 3 + 9 files changed, 768 insertions(+) create mode 100644 scripts/idl/BUILD.gn create mode 100644 scripts/idl/__init__.py create mode 100644 scripts/idl/matter_grammar.lark create mode 100755 scripts/idl/matter_idl_parser.py create mode 100644 scripts/idl/matter_idl_types.py create mode 100644 scripts/idl/setup.py create mode 100755 scripts/idl/test_matter_idl_parser.py diff --git a/BUILD.gn b/BUILD.gn index 258ea4852ff27f..0fe2b8cba7a38f 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -160,6 +160,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") { deps = [ "//:fake_platform_tests", "//scripts/build:build_examples.tests", + "//scripts/idl:idl.tests", "//src:tests_run", ] } diff --git a/scripts/idl/BUILD.gn b/scripts/idl/BUILD.gn new file mode 100644 index 00000000000000..c771fa4cf6bfb1 --- /dev/null +++ b/scripts/idl/BUILD.gn @@ -0,0 +1,35 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python.gni") + +pw_python_package("idl") { + setup = [ "setup.py" ] + inputs = [ + # Dependency grammar + "matter_grammar.lark", + ] + + sources = [ + "__init__.py", + "matter_idl_parser.py", + "matter_idl_types.py", + ] + + tests = [ "test_matter_idl_parser.py" ] +} diff --git a/scripts/idl/__init__.py b/scripts/idl/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/scripts/idl/matter_grammar.lark b/scripts/idl/matter_grammar.lark new file mode 100644 index 00000000000000..b7c388bcb9c061 --- /dev/null +++ b/scripts/idl/matter_grammar.lark @@ -0,0 +1,59 @@ +struct: "struct"i id "{" struct_field* "}" +enum: "enum"i id ":" data_type "{" enum_entry* "}" + +event: event_priority "event"i id "=" number "{" struct_field* "}" + +?event_priority: "critical"i -> critical_priority + | "info"i -> info_priority + | "debug"i -> debug_priority + +attribute: attribute_tag* "attribute"i field +attribute_tag: "readonly"i -> attr_readonly + | "global"i -> attr_global + +request_struct: "request"i struct +response_struct: "response"i struct + +command: "command"i id "(" id? ")" ":" id "=" number ";" + +cluster: cluster_side "cluster"i id "=" number "{" (enum|event|attribute|struct|request_struct|response_struct|command)* "}" +?cluster_side: "server"i -> server_cluster + | "client"i -> client_cluster + +endpoint: "endpoint"i number "{" endpoint_cluster* "}" +endpoint_cluster: endpoint_cluster_type "cluster"i id ";" + +?endpoint_cluster_type: "server"i -> endpoint_server_cluster + | "binding"i -> endpoint_binding_to_cluster + +enum_entry: id "=" number ";" +number: POSITIVE_INTEGER | HEX_INTEGER + +struct_field: member_attribute* field + +member_attribute: "optional"i -> optional + | "nullable"i -> nullable + +field: data_type id list_marker? "=" number ";" +list_marker: "[" "]" + +data_type: type ("<" number ">")? + +id: ID +type: ID + +COMMENT: "{" /(.|\n)+/ "}" + | "//" /.*/ + +POSITIVE_INTEGER: /\d+/ +HEX_INTEGER: /0x[A-Fa-f0-9]+/ +ID: /[a-zA-Z_][a-zA-Z0-9_]*/ + +idl: (struct|enum|cluster|endpoint)* + +%import common.WS +%import common.C_COMMENT +%import common.CPP_COMMENT +%ignore WS +%ignore C_COMMENT +%ignore CPP_COMMENT diff --git a/scripts/idl/matter_idl_parser.py b/scripts/idl/matter_idl_parser.py new file mode 100755 index 00000000000000..29b71d0caa62fb --- /dev/null +++ b/scripts/idl/matter_idl_parser.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python + +import logging + +from lark import Lark +from lark.visitors import Transformer, v_args + +try: + from .matter_idl_types import * +except: + import os + import sys + sys.path.append(os.path.abspath(os.path.dirname(__file__))) + + from matter_idl_types import * + + +class MatterIdlTransformer(Transformer): + """A transformer capable to transform data + parsed by Lark according to matter_grammar.lark + """ + + def number(self, tokens): + """Numbers in the grammar are integers or hex numbers. + """ + if len(tokens) != 1: + raise Error("Unexpected argument counts") + + n = tokens[0].value + if n.startswith('0x'): + return int(n[2:], 16) + else: + return int(n) + + def id(self, tokens): + """An id is a string containing an identifier + """ + if len(tokens) != 1: + raise Error("Unexpected argument counts") + return tokens[0].value + + def type(self, tokens): + """A type is just a string for the type + """ + if len(tokens) != 1: + raise Error("Unexpected argument counts") + return tokens[0].value + + def data_type(self, tokens): + if len(tokens) == 1: + return DataType(name=tokens[0]) + # Just a string for data type + elif len(tokens) == 2: + return DataType(name=tokens[0], max_length=tokens[1]) + else: + raise Error("Unexpected size for data type") + + @v_args(inline=True) + def enum_entry(self, id, number): + return EnumEntry(name=id, code=number) + + @v_args(inline=True) + def enum(self, id, type, *entries): + return Enum(name=id, base_type=type, entries=list(entries)) + + def field(self, args): + data_type, name = args[0], args[1] + is_list = (len(args) == 4) + code = args[-1] + + return Field(data_type=data_type, name=name, code=code, is_list=is_list) + + def optional(self, _): + return FieldAttribute.OPTIONAL + + def nullable(self, _): + return FieldAttribute.NULLABLE + + def attr_readonly(self, _): + return AttributeTag.READABLE + + def attr_global(self, _): + return AttributeTag.GLOBAL + + def critical_priority(self, _): + return EventPriority.CRITICAL + + def info_priority(self, _): + return EventPriority.INFO + + def debug_priority(self, _): + return EventPriority.DEBUG + + def endpoint_server_cluster(self, _): + return EndpointContentType.SERVER_CLUSTER + + def endpoint_binding_to_cluster(self, _): + return EndpointContentType.CLIENT_BINDING + + def struct_field(self, args): + # Last argument is the named_member, the rest + # are attributes + field = args[-1] + field.attributes = set(args[:-1]) + return field + + def server_cluster(self, _): + return ClusterSide.SERVER + + def client_cluster(self, _): + return ClusterSide.CLIENT + + def command(self, args): + # A command has 3 arguments if no input or + # 4 arguments if input parameter is available + param_in = None + if len(args) > 3: + param_in = args[1] + return Command(name=args[0], input_param=param_in, output_param=args[-2], code=args[-1]) + + def event(self, args): + return Event(priority=args[0], name=args[1], code=args[2], fields=args[3:], ) + + def attribute(self, args): + tags = set(args[:-1]) + # until we support write only (and need a bit of a reshuffle) + # if the 'attr_readonly == READABLE' is not in the list, we make things + # read/write + if AttributeTag.READABLE not in tags: + tags.add(AttributeTag.READABLE) + tags.add(AttributeTag.WRITABLE) + + return Attribute(definition=args[-1], tags=tags) + + @v_args(inline=True) + def struct(self, id, *fields): + return Struct(name=id, fields=list(fields)) + + @v_args(inline=True) + def request_struct(self, value): + value.tag = StructTag.REQUEST + return value + + @v_args(inline=True) + def response_struct(self, value): + value.tag = StructTag.RESPONSE + return value + + @v_args(inline=True) + def endpoint(self, number, *clusters): + endpoint = Endpoint(number=number) + + for t, name in clusters: + if t == EndpointContentType.CLIENT_BINDING: + endpoint.client_bindings.append(name) + elif t == EndpointContentType.SERVER_CLUSTER: + endpoint.server_clusters.append(name) + else: + raise Error("Unknown endpoint content: %r" % t) + + return endpoint + + @v_args(inline=True) + def endpoint_cluster(self, t, id): + return (t, id) + + @v_args(inline=True) + def cluster(self, side, name, code, *content): + result = Cluster(side=side, name=name, code=code) + + for item in content: + if type(item) == Enum: + result.enums.append(item) + elif type(item) == Event: + result.events.append(item) + elif type(item) == Attribute: + result.attributes.append(item) + elif type(item) == Struct: + result.structs.append(item) + elif type(item) == Command: + result.commands.append(item) + else: + raise Error("UNKNOWN cluster content item: %r" % item) + + return result + + def idl(self, items): + idl = Idl() + + for item in items: + if type(item) == Enum: + idl.enums.append(item) + elif type(item) == Struct: + idl.structs.append(item) + elif type(item) == Cluster: + idl.clusters.append(item) + elif type(item) == Endpoint: + idl.endpoints.append(item) + else: + raise Error("UNKNOWN idl content item: %r" % item) + + return idl + + +def CreateParser(): + return Lark.open('matter_grammar.lark', rel_to=__file__, start='idl', parser='lalr', transformer=MatterIdlTransformer()) + + +if __name__ == '__main__': + import click + import coloredlogs + + # Supported log levels, mapping string values required for argument + # parsing into logging constants + __LOG_LEVELS__ = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'fatal': logging.FATAL, + } + + @click.command() + @click.option( + '--log-level', + default='INFO', + type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), + help='Determines the verbosity of script output.') + @click.argument('filename') + def main(log_level, filename=None): + coloredlogs.install(level=__LOG_LEVELS__[ + log_level], fmt='%(asctime)s %(levelname)-7s %(message)s') + + logging.info("Starting to parse ...") + data = CreateParser().parse(open(filename).read()) + logging.info("Parse completed") + + logging.info("Data:") + print(data) + + main() diff --git a/scripts/idl/matter_idl_types.py b/scripts/idl/matter_idl_types.py new file mode 100644 index 00000000000000..587a45232068ba --- /dev/null +++ b/scripts/idl/matter_idl_types.py @@ -0,0 +1,135 @@ +import enum + +from dataclasses import dataclass, field +from typing import List, Set, Union + + +class FieldAttribute(enum.Enum): + OPTIONAL = enum.auto() + NULLABLE = enum.auto() + + +class AttributeTag(enum.Enum): + READABLE = enum.auto() + WRITABLE = enum.auto() + GLOBAL = enum.auto() + + +class EventPriority(enum.Enum): + DEBUG = enum.auto() + INFO = enum.auto() + CRITICAL = enum.auto() + + +class ClusterSide(enum.Enum): + CLIENT = enum.auto() + SERVER = enum.auto() + + +class StructTag(enum.Enum): + REQUEST = enum.auto() + RESPONSE = enum.auto() + + +class EndpointContentType(enum.Enum): + SERVER_CLUSTER = enum.auto() + CLIENT_BINDING = enum.auto() + + +@dataclass +class DataType: + name: str + + # Applies for strings (char or binary) + max_length: Union[int, None] = None + + +@dataclass +class Field: + data_type: str + code: int + name: str + is_list: bool = False + attributes: Set[FieldAttribute] = field(default_factory=set) + + +@dataclass +class Attribute: + definition: Field + tags: Set[AttributeTag] = field(default_factory=set()) + + @property + def is_readable(self): + return AttributeTag.READABLE in self.tags + + @property + def is_writable(self): + return AttributeTag.WRITABLE in self.tags + + @property + def is_global(self): + return AttributeTag.GLOBAL in self.tags + + +@dataclass +class Struct: + name: str + fields: List[Field] + tag: Union[StructTag, None] = None + + +@dataclass +class Event: + priority: EventPriority + name: str + code: int + fields: List[Field] + + +@dataclass +class EnumEntry: + name: str + code: int + + +@dataclass +class Enum: + name: str + base_type: str + entries: List[EnumEntry] + + +@dataclass +class Command: + name: str + code: int + input_param: str + output_param: str + + +@dataclass +class Cluster: + side: ClusterSide + name: str + code: int + enums: List[Enum] = field(default_factory=list) + events: List[Event] = field(default_factory=list) + attributes: List[Attribute] = field(default_factory=list) + structs: List[Struct] = field(default_factory=list) + commands: List[Command] = field(default_factory=list) + + +@dataclass +class Endpoint: + number: int + server_clusters: List[str] = field(default_factory=list) + client_bindings: List[str] = field(default_factory=list) + + +@dataclass +class Idl: + # Enums and structs represent globally used items + enums: List[Enum] = field(default_factory=list) + structs: List[Struct] = field(default_factory=list) + clusters: List[Cluster] = field(default_factory=list) + endpoints: List[Endpoint] = field(default_factory=list) diff --git a/scripts/idl/setup.py b/scripts/idl/setup.py new file mode 100644 index 00000000000000..4566792c40074a --- /dev/null +++ b/scripts/idl/setup.py @@ -0,0 +1,28 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""The idl package.""" + +import setuptools # type: ignore + +setuptools.setup( + name='idl', + version='0.0.1', + author='Project CHIP Authors', + description='Parse matter idl files', + packages=setuptools.find_packages(), + package_data={'idl': ['py.typed']}, + zip_safe=False, +) diff --git a/scripts/idl/test_matter_idl_parser.py b/scripts/idl/test_matter_idl_parser.py new file mode 100755 index 00000000000000..7d841f09c51bb1 --- /dev/null +++ b/scripts/idl/test_matter_idl_parser.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from .matter_idl_parser import CreateParser + from .matter_idl_types import * +except: + import os + import sys + sys.path.append(os.path.abspath(os.path.dirname(__file__))) + + from matter_idl_parser import CreateParser + from matter_idl_types import * + +import unittest + + +def parseText(txt): + return CreateParser().parse(txt) + + +class TestParser(unittest.TestCase): + + def test_skips_comments(self): + actual = parseText(""" + // this is a single line comment + // repeated + + /* This is a C++ comment + and also whitespace should be ignored + */ + """) + expected = Idl() + + self.assertEqual(actual, expected) + + def test_global_enum(self): + actual = parseText(""" + enum GlobalEnum : ENUM8 { + kValue1 = 1; + kOther = 0x12; /* hex numbers tested sporadically */ + } + """) + + expected = Idl(enums=[ + Enum(name='GlobalEnum', base_type=DataType(name='ENUM8'), + entries=[ + EnumEntry(name="kValue1", code=1), + EnumEntry(name="kOther", code=0x12), + ])] + ) + self.assertEqual(actual, expected) + + def test_global_struct(self): + actual = parseText(""" + struct Something { + CHAR_STRING astring = 1; + optional CLUSTER_ID idlist[] = 2; + nullable int valueThatIsNullable = 0x123; + char_string<123> sized_string = 222; + } + """) + + expected = Idl(structs=[ + Struct(name='Something', + fields=[ + Field( + data_type=DataType(name="CHAR_STRING"), code=1, name="astring", ), + Field(data_type=DataType(name="CLUSTER_ID"), code=2, name="idlist", is_list=True, attributes=set( + [FieldAttribute.OPTIONAL])), + Field(data_type=DataType(name="int"), code=0x123, name="valueThatIsNullable", attributes=set( + [FieldAttribute.NULLABLE])), + Field(data_type=DataType(name="char_string", max_length=123), + code=222, name="sized_string", attributes=set()), + ])] + ) + self.assertEqual(actual, expected) + + def test_cluster_attribute(self): + actual = parseText(""" + server cluster MyCluster = 0x321 { + readonly attribute int8u roAttr = 1; + attribute int32u rwAttr[] = 123; + global attribute int32u grwAttr[] = 124; + readonly global attribute int32u groAttr[] = 125; + } + """) + + expected = Idl(clusters=[ + Cluster(side=ClusterSide.SERVER, + name="MyCluster", + code=0x321, + attributes=[ + Attribute(tags=set([AttributeTag.READABLE]), definition=Field( + data_type=DataType(name="int8u"), code=1, name="roAttr")), + Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field( + data_type=DataType(name="int32u"), code=123, name="rwAttr", is_list=True)), + Attribute(tags=set([AttributeTag.GLOBAL, AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field( + data_type=DataType(name="int32u"), code=124, name="grwAttr", is_list=True)), + Attribute(tags=set([AttributeTag.GLOBAL, AttributeTag.READABLE]), definition=Field( + data_type=DataType(name="int32u"), code=125, name="groAttr", is_list=True)), + ] + )]) + self.assertEqual(actual, expected) + + def test_sized_attribute(self): + actual = parseText(""" + server cluster MyCluster = 1 { + attribute char_string<11> attr1 = 1; + attribute octet_string<33> attr2[] = 2; + } + """) + + expected = Idl(clusters=[ + Cluster(side=ClusterSide.SERVER, + name="MyCluster", + code=1, + attributes=[ + Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field( + data_type=DataType(name="char_string", max_length=11), code=1, name="attr1")), + Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field( + data_type=DataType(name="octet_string", max_length=33), code=2, name="attr2", is_list=True)), + ] + )]) + self.assertEqual(actual, expected) + + def test_cluster_commands(self): + actual = parseText(""" + server cluster WithCommands = 1 { + struct FreeStruct {} + request struct InParam {} + response struct OutParam {} + + command WithoutArg(): DefaultSuccess = 123; + command InOutStuff(InParam): OutParam = 222; + } + """) + expected = Idl(clusters=[ + Cluster(side=ClusterSide.SERVER, + name="WithCommands", + code=1, + structs=[ + Struct(name="FreeStruct", fields=[]), + Struct(name="InParam", fields=[], + tag=StructTag.REQUEST), + Struct(name="OutParam", fields=[], + tag=StructTag.RESPONSE), + ], + commands=[ + Command(name="WithoutArg", code=123, + input_param=None, output_param="DefaultSuccess"), + Command(name="InOutStuff", code=222, + input_param="InParam", output_param="OutParam"), + ], + )]) + self.assertEqual(actual, expected) + + def test_cluster_enum(self): + actual = parseText(""" + client cluster WithEnums = 0xab { + enum TestEnum : ENUM16 { + A = 0x123; + B = 0x234; + } + } + """) + expected = Idl(clusters=[ + Cluster(side=ClusterSide.CLIENT, + name="WithEnums", + code=0xab, + enums=[ + Enum(name="TestEnum", base_type=DataType(name="ENUM16"), + entries=[ + EnumEntry(name="A", code=0x123), + EnumEntry(name="B", code=0x234), + ])], + )]) + self.assertEqual(actual, expected) + + def test_cluster_events(self): + actual = parseText(""" + client cluster EventTester = 0x123 { + critical event StartUp = 0 { + INT32U softwareVersion = 0; + } + info event Hello = 1 {} + debug event GoodBye = 2 {} + } + """) + expected = Idl(clusters=[ + Cluster(side=ClusterSide.CLIENT, + name="EventTester", + code=0x123, + events=[ + Event(priority=EventPriority.CRITICAL, name="StartUp", code=0, fields=[ + Field(data_type=DataType(name="INT32U"), + code=0, name="softwareVersion"), + ]), + Event(priority=EventPriority.INFO, + name="Hello", code=1, fields=[]), + Event(priority=EventPriority.DEBUG, + name="GoodBye", code=2, fields=[]), + ])]) + self.assertEqual(actual, expected) + + def test_multiple_clusters(self): + actual = parseText(""" + server cluster A = 1 {} + client cluster B = 2 {} + client cluster C = 3 {} + """) + + expected = Idl(clusters=[ + Cluster(side=ClusterSide.SERVER, name="A", code=1), + Cluster(side=ClusterSide.CLIENT, name="B", code=2), + Cluster(side=ClusterSide.CLIENT, name="C", code=3), + ]) + self.assertEqual(actual, expected) + + def test_endpoints(self): + actual = parseText(""" + endpoint 12 { + server cluster Foo; + server cluster Bar; + binding cluster Bar; + binding cluster Test; + } + """) + + expected = Idl(endpoints=[Endpoint(number=12, + server_clusters=["Foo", "Bar"], + client_bindings=["Bar", "Test"],) + ]) + self.assertEqual(actual, expected) + + def test_multi_endpoints(self): + actual = parseText(""" + endpoint 1 {} + endpoint 2 {} + endpoint 0xa {} + endpoint 100 {} + """) + + expected = Idl(endpoints=[ + Endpoint(number=1), + Endpoint(number=2), + Endpoint(number=10), + Endpoint(number=100), + ]) + self.assertEqual(actual, expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/requirements.txt b/scripts/requirements.txt index f68b124bb7d6f7..0be3b90c84f2e9 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -46,3 +46,6 @@ pandas ; platform_machine != 'aarch64' and platform_machine != 'arm64' # scripts/build click + +# scripts/idl +lark