From aaa2b6c7c4f612f4537e25e1188dedc78cb01db3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 09:58:46 -0400 Subject: [PATCH 01/36] Start creating a IDL codegen so we can self-test parsed output --- .../matter_idl/generators/idl/MatterIdl.jinja | 1 + .../matter_idl/generators/idl/__init__.py | 43 +++++++++++++++++++ .../matter_idl/generators/registry.py | 5 +++ 3 files changed, 49 insertions(+) create mode 100644 scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja create mode 100644 scripts/py_matter_idl/matter_idl/generators/idl/__init__.py diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja new file mode 100644 index 00000000000000..5ae8a4d69aaa2a --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -0,0 +1 @@ +FIXME: implement diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py new file mode 100644 index 00000000000000..f5f5fd3046f3a5 --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import List + +from matter_idl.generators import CodeGenerator, GeneratorStorage +from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl + + +class IdlGenerator(CodeGenerator): + """ + Generation .matter idl files for a given IDL + """ + + def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): + super().__init__(storage, idl, fs_loader_searchpath=os.path.dirname(__file__)) + + def internal_render_all(self): + """ + Renders the output. + """ + + # Header containing a macro to initialize all cluster plugins + self.internal_render_one_output( + template_path="MatterIdl.jinja", + output_file_name="idl.matter", + vars={ + 'idl': self.idl + } + ) + diff --git a/scripts/py_matter_idl/matter_idl/generators/registry.py b/scripts/py_matter_idl/matter_idl/generators/registry.py index e3c8e1ed32397e..b02b9bbcf12f0b 100644 --- a/scripts/py_matter_idl/matter_idl/generators/registry.py +++ b/scripts/py_matter_idl/matter_idl/generators/registry.py @@ -17,6 +17,7 @@ from matter_idl.generators.cpp.application import CppApplicationGenerator from matter_idl.generators.cpp.tlvmeta import TLVMetaDataGenerator +from matter_idl.generators.idl import IdlGenerator from matter_idl.generators.java import JavaClassGenerator, JavaJNIGenerator @@ -30,6 +31,7 @@ class CodeGenerator(enum.Enum): JAVA_CLASS = enum.auto() CPP_APPLICATION = enum.auto() CPP_TLVMETA = enum.auto() + IDL = enum.auto() CUSTOM = enum.auto() def Create(self, *args, **kargs): @@ -41,6 +43,8 @@ def Create(self, *args, **kargs): return CppApplicationGenerator(*args, **kargs) elif self == CodeGenerator.CPP_TLVMETA: return TLVMetaDataGenerator(*args, **kargs) + elif self == CodeGenerator.IDL: + return IdlGenerator(*args, **kargs) elif self == CodeGenerator.CUSTOM: # Use a package naming convention to find the custom generator: # ./matter_idl_plugin/__init__.py defines a subclass of CodeGenerator named CustomGenerator. @@ -70,5 +74,6 @@ def FromString(name): 'java-class': CodeGenerator.JAVA_CLASS, 'cpp-app': CodeGenerator.CPP_APPLICATION, 'cpp-tlvmeta': CodeGenerator.CPP_TLVMETA, + 'idl': CodeGenerator.IDL, 'custom': CodeGenerator.CUSTOM, } From 672ecf3d5cea64568d0c7512db925ec26abe8ea2 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 10:13:04 -0400 Subject: [PATCH 02/36] Start with listing clusters --- .../matter_idl/generators/idl/MatterIdl.jinja | 18 +++++++++++++++++- .../matter_idl/generators/idl/__init__.py | 13 +++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 5ae8a4d69aaa2a..bda6186a185367 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -1 +1,17 @@ -FIXME: implement +// This IDL was auto-generated from a parsed data structure + +{% for cluster in idl.clusters -%} +{{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { + // FIXME: implement +} + +{% endfor %} + +{%- if idl.endpoints %} +{% for endpoint in idl.endpoints %} + +endpoint {{endpoint.number}} { + // FIXME: report endpoint +} +{% endfor %} +{% endif %} diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index f5f5fd3046f3a5..4566ca941a4dd3 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -18,6 +18,17 @@ from matter_idl.generators import CodeGenerator, GeneratorStorage from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl +def HumanTextString(value: ClusterSide) -> str: + if type(value) is ClusterSide: + if value == ClusterSide.CLIENT: + return "client" + if value == ClusterSide.SERVER: + return "server" + + + # wrong value in general + return "Unknown/unsupported: %r" % value + class IdlGenerator(CodeGenerator): """ @@ -27,6 +38,8 @@ class IdlGenerator(CodeGenerator): def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): super().__init__(storage, idl, fs_loader_searchpath=os.path.dirname(__file__)) + self.jinja_env.filters['idltxt'] = HumanTextString + def internal_render_all(self): """ Renders the output. From 6cb81c35a9f5efb0dc6878824cbce57e8de8ac19 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 10:18:29 -0400 Subject: [PATCH 03/36] Enum listing --- .../matter_idl/generators/idl/MatterIdl.jinja | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index bda6186a185367..71fd759bf67b0c 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -2,6 +2,14 @@ {% for cluster in idl.clusters -%} {{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { + {%- for enum in cluster.enums %} + enum {{enum.name}} : {{ enum.base_type}} { + {%- for entry in enum.entries %} + {{entry.name}} = {{entry.code}}; + {%- endfor %} + } + + {%- endfor %} // FIXME: implement } From 41da54f66bf94a4026ff5ae91ea01f66f0de8913 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 10:47:25 -0400 Subject: [PATCH 04/36] A lot more things supported --- .../matter_idl/generators/idl/MatterIdl.jinja | 42 ++++++++++++- .../matter_idl/generators/idl/__init__.py | 61 +++++++++++++++++-- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 71fd759bf67b0c..37f12beef371aa 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -1,16 +1,56 @@ +{% macro render_field(field) -%} +{%- if field.qualities %}{{field.qualities | idltxt}} {% endif -%} +{{field.data_type.name}} {{field.name}} +{%- if field.is_list %}[]{% endif -%} +{##} = {{field.code}}; +{%- endmacro %} // This IDL was auto-generated from a parsed data structure {% for cluster in idl.clusters -%} {{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { {%- for enum in cluster.enums %} + enum {{enum.name}} : {{ enum.base_type}} { {%- for entry in enum.entries %} {{entry.name}} = {{entry.code}}; {%- endfor %} } + {%- endfor %} + {%- for bitmap in cluster.bitmaps %} + bitmap {{bitmap.name}} : {{ bitmap.base_type}} { + {%- for entry in bitmap.entries %} + {{entry.name}} = {{entry.code}}; + {%- endfor %} + } {%- endfor %} - // FIXME: implement + {%- for s in cluster.structs %} + + {% if s.tag %}{{s.tag | idltxt}} {% endif -%} + {% if s.qualities %}{{s.qualities | idltxt}} {% endif -%} + struct {{s.name}} {##} + {%- if s.code %}= {{s.code}} {% endif -%} + { + {%- for field in s.fields %} + {{render_field(field)}} + {%- endfor %} + } + {%- endfor %} + {%- for e in cluster.events %} + + {% if e.qualities %}{{e.qualities | idltxt}} {% endif -%} + {{e.priority | idltxt}} event {{e | event_access}}{{e.name}} = {{e.code}} { + {%- for field in e.fields %} + {{render_field(field)}} + {%- endfor %} + } + {%- endfor %} + + + // FIXME: + // - events + // - attributes + // - commands } {% endfor %} diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 4566ca941a4dd3..d8250f9ec0f3ed 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -16,20 +16,72 @@ from typing import List from matter_idl.generators import CodeGenerator, GeneratorStorage -from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl +from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl, StructTag, FieldQuality, StructQuality, EventPriority, EventQuality, Event, AccessPrivilege -def HumanTextString(value: ClusterSide) -> str: +def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|EventQuality|AccessPrivilege) -> str: if type(value) is ClusterSide: if value == ClusterSide.CLIENT: return "client" if value == ClusterSide.SERVER: return "server" - + elif type(value) is StructTag: + if value == StructTag.REQUEST: + return "request" + if value == StructTag.RESPONSE: + return "response" + elif type(value) is FieldQuality: + result = "" + if FieldQuality.OPTIONAL in value: + result += "optional " + if FieldQuality.NULLABLE in value: + result += "nullable " + if FieldQuality.FABRIC_SENSITIVE in value: + result += "fabric_sensitive " + return result.strip() + elif type(value) is StructQuality: + result = "" + if value == StructQuality.FABRIC_SCOPED: + result += "fabric_scoped " + return result.strip() + elif type(value) is EventPriority: + if value == EventPriority.DEBUG: + return "debug" + if value == EventPriority.INFO: + return "info" + if value == EventPriority.CRITICAL: + return "critical" + elif type(value) is EventQuality: + result = "" + if EventQuality.FABRIC_SENSITIVE in value: + result += "fabric_sensitive " + return result.strip() + elif type(value) is AccessPrivilege: + if value == AccessPrivilege.VIEW: + return "view" + if value == AccessPrivilege.OPERATE: + return "operate" + if value == AccessPrivilege.MANAGE: + return "manage" + if value == AccessPrivilege.ADMINISTER: + return "administer" # wrong value in general return "Unknown/unsupported: %r" % value +def event_access_string(e: Event) -> str: + """Generates the access string required for an event. If string is non-empty it will + include a trailing space + """ + result = "" + if e.readacl != AccessPrivilege.VIEW: + result += "read: " + human_text_string(e.readacl) + + if not result: + return "" + return f"access({result}) " + + class IdlGenerator(CodeGenerator): """ Generation .matter idl files for a given IDL @@ -38,7 +90,8 @@ class IdlGenerator(CodeGenerator): def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): super().__init__(storage, idl, fs_loader_searchpath=os.path.dirname(__file__)) - self.jinja_env.filters['idltxt'] = HumanTextString + self.jinja_env.filters['idltxt'] = human_text_string + self.jinja_env.filters['event_access'] = event_access_string def internal_render_all(self): """ From 3d11d544a34e0d0498a69d3f63e1d91096cd4557 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:04:37 -0400 Subject: [PATCH 05/36] Attribute rendering --- .../matter_idl/generators/idl/MatterIdl.jinja | 54 +++++++++++++------ .../matter_idl/generators/idl/__init__.py | 18 ++++++- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 37f12beef371aa..7fe855cfd504df 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -1,9 +1,34 @@ -{% macro render_field(field) -%} +{% macro render_field(field) -%}{# + Macro for the output of a single field entry such as: + + int16u identifyTime = 0; + optional int16u transitionTime = 3; + optional nullable int16u transitionTime = 2; + optional ExtensionFieldSet extensionFieldSets[] = 5; +#} + {%- if field.qualities %}{{field.qualities | idltxt}} {% endif -%} {{field.data_type.name}} {{field.name}} {%- if field.is_list %}[]{% endif -%} {##} = {{field.code}}; -{%- endmacro %} +{%- endmacro -%} + +{% macro render_struct(s) -%}{# + Macro for the output of a complete struct +#} + {%- if s.tag %}{{s.tag | idltxt}} {% endif -%} + {% if s.qualities %}{{s.qualities | idltxt}} {% endif -%} + struct {{s.name}} {##} + {%- if s.code %}= {{s.code}} {% endif -%} + { + {%- for field in s.fields %} + {{render_field(field)}} + {%- endfor %} + } + +{%- endmacro -%} + + // This IDL was auto-generated from a parsed data structure {% for cluster in idl.clusters -%} @@ -24,18 +49,10 @@ {%- endfor %} } {%- endfor %} - {%- for s in cluster.structs %} + {% for s in cluster.structs | rejectattr("tag") -%} + {{render_struct(s)}} - {% if s.tag %}{{s.tag | idltxt}} {% endif -%} - {% if s.qualities %}{{s.qualities | idltxt}} {% endif -%} - struct {{s.name}} {##} - {%- if s.code %}= {{s.code}} {% endif -%} - { - {%- for field in s.fields %} - {{render_field(field)}} - {%- endfor %} - } - {%- endfor %} + {% endfor %} {%- for e in cluster.events %} {% if e.qualities %}{{e.qualities | idltxt}} {% endif -%} @@ -45,11 +62,18 @@ {%- endfor %} } {%- endfor %} + {%- for a in cluster.attributes -%} + {{a.qualities | idltxt}}attribute {{render_field(a.definition)}} + {%- if loop.last %} + + {% endif %} + {% endfor %} + {%- for s in cluster.structs | selectattr("tag") -%} + {{render_struct(s)}} + {% endfor %} // FIXME: - // - events - // - attributes // - commands } diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index d8250f9ec0f3ed..e5e2c50f14c4f7 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -16,9 +16,9 @@ from typing import List from matter_idl.generators import CodeGenerator, GeneratorStorage -from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl, StructTag, FieldQuality, StructQuality, EventPriority, EventQuality, Event, AccessPrivilege +from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl, StructTag, FieldQuality, StructQuality, EventPriority, EventQuality, Event, AccessPrivilege, Struct, AttributeQuality -def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|EventQuality|AccessPrivilege) -> str: +def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|EventQuality|AccessPrivilege|AttributeQuality) -> str: if type(value) is ClusterSide: if value == ClusterSide.CLIENT: return "client" @@ -64,6 +64,15 @@ def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|E return "manage" if value == AccessPrivilege.ADMINISTER: return "administer" + elif type(value) is AttributeQuality: + result = "" + if AttributeQuality.TIMED_WRITE in value: + result += "timedwrite " + if AttributeQuality.WRITABLE not in value: + result += "readonly " + if AttributeQuality.NOSUBSCRIBE in value: + result += "nosubscribe " + return result # wrong value in general return "Unknown/unsupported: %r" % value @@ -81,6 +90,9 @@ def event_access_string(e: Event) -> str: return "" return f"access({result}) " +def is_untagged_struct(s: Struct): + return s.tag is None + class IdlGenerator(CodeGenerator): """ @@ -93,6 +105,8 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): self.jinja_env.filters['idltxt'] = human_text_string self.jinja_env.filters['event_access'] = event_access_string + self.jinja_env.tests['is_untagged_struct'] = is_untagged_struct + def internal_render_all(self): """ Renders the output. From 3e3e7ce62db142e8e38ad0c8c87960b5dd1e1d36 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:07:53 -0400 Subject: [PATCH 06/36] Support for string and octet string sizes --- .../matter_idl/generators/idl/MatterIdl.jinja | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 7fe855cfd504df..fd4bdc5d6d89cb 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -8,7 +8,9 @@ #} {%- if field.qualities %}{{field.qualities | idltxt}} {% endif -%} -{{field.data_type.name}} {{field.name}} +{{field.data_type.name}} +{%- if field.data_type.max_length -%} <{{field.data_type.max_length}}> {%- endif -%} +{##} {{field.name}} {%- if field.is_list %}[]{% endif -%} {##} = {{field.code}}; {%- endmacro -%} @@ -62,12 +64,12 @@ {%- endfor %} } {%- endfor %} - {%- for a in cluster.attributes -%} - {{a.qualities | idltxt}}attribute {{render_field(a.definition)}} + {%- for a in cluster.attributes %} + {{a.qualities | idltxt}}attribute {{render_field(a.definition)}} {%- if loop.last %} {% endif %} - {% endfor %} + {%- endfor %} {%- for s in cluster.structs | selectattr("tag") -%} {{render_struct(s)}} From 07d15e637f09ec356b4c3a2355c9bebad31bf81c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:14:04 -0400 Subject: [PATCH 07/36] Timed command support --- .../matter_idl/generators/idl/MatterIdl.jinja | 6 ++--- .../matter_idl/generators/idl/__init__.py | 26 +++++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index fd4bdc5d6d89cb..c44b907be7d07b 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -74,9 +74,9 @@ {{render_struct(s)}} {% endfor %} - - // FIXME: - // - commands + {% for c in cluster.commands -%} + {{c.qualities | idltxt}}command {{c | command_access}}{{c.name}}({{c.input_param}}): {{c.output_param}} = {{c.code}}; + {% endfor %} } {% endfor %} diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index e5e2c50f14c4f7..916fd7be5ece30 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -16,9 +16,9 @@ from typing import List from matter_idl.generators import CodeGenerator, GeneratorStorage -from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl, StructTag, FieldQuality, StructQuality, EventPriority, EventQuality, Event, AccessPrivilege, Struct, AttributeQuality +from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl, StructTag, FieldQuality, StructQuality, EventPriority, EventQuality, Event, AccessPrivilege, Struct, AttributeQuality, Command, CommandQuality -def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|EventQuality|AccessPrivilege|AttributeQuality) -> str: +def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|EventQuality|AccessPrivilege|AttributeQuality|CommandQuality) -> str: if type(value) is ClusterSide: if value == ClusterSide.CLIENT: return "client" @@ -73,6 +73,13 @@ def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|E if AttributeQuality.NOSUBSCRIBE in value: result += "nosubscribe " return result + elif type(value) is CommandQuality: + result = "" + if CommandQuality.TIMED_INVOKE in value: + result += "timed " + if CommandQuality.FABRIC_SCOPED in value: + result += "fabric " + return result # wrong value in general return "Unknown/unsupported: %r" % value @@ -90,8 +97,17 @@ def event_access_string(e: Event) -> str: return "" return f"access({result}) " -def is_untagged_struct(s: Struct): - return s.tag is None +def command_access_string(c: Command) -> str: + """Generates the access string required for an event. If string is non-empty it will + include a trailing space + """ + result = "" + if c.invokeacl != AccessPrivilege.OPERATE: + result += "invoke: " + human_text_string(c.invokeacl) + + if not result: + return "" + return f"access({result}) " class IdlGenerator(CodeGenerator): @@ -104,8 +120,8 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): self.jinja_env.filters['idltxt'] = human_text_string self.jinja_env.filters['event_access'] = event_access_string + self.jinja_env.filters['command_access'] = command_access_string - self.jinja_env.tests['is_untagged_struct'] = is_untagged_struct def internal_render_all(self): """ From 3a2133d97804aaa3a53314c5ebf80d27744d5b53 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:15:13 -0400 Subject: [PATCH 08/36] Restyle --- .../matter_idl/generators/idl/__init__.py | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 916fd7be5ece30..675a1133052b26 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -16,9 +16,11 @@ from typing import List from matter_idl.generators import CodeGenerator, GeneratorStorage -from matter_idl.matter_idl_types import Cluster, ClusterSide, Idl, StructTag, FieldQuality, StructQuality, EventPriority, EventQuality, Event, AccessPrivilege, Struct, AttributeQuality, Command, CommandQuality +from matter_idl.matter_idl_types import (AccessPrivilege, AttributeQuality, Cluster, ClusterSide, Command, CommandQuality, Event, + EventPriority, EventQuality, FieldQuality, Idl, Struct, StructQuality, StructTag) -def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|EventQuality|AccessPrivilege|AttributeQuality|CommandQuality) -> str: + +def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality) -> str: if type(value) is ClusterSide: if value == ClusterSide.CLIENT: return "client" @@ -86,28 +88,29 @@ def human_text_string(value: ClusterSide|StructTag|StructQuality|EventPriority|E def event_access_string(e: Event) -> str: - """Generates the access string required for an event. If string is non-empty it will - include a trailing space - """ - result = "" - if e.readacl != AccessPrivilege.VIEW: - result += "read: " + human_text_string(e.readacl) + """Generates the access string required for an event. If string is non-empty it will + include a trailing space + """ + result = "" + if e.readacl != AccessPrivilege.VIEW: + result += "read: " + human_text_string(e.readacl) + + if not result: + return "" + return f"access({result}) " - if not result: - return "" - return f"access({result}) " def command_access_string(c: Command) -> str: - """Generates the access string required for an event. If string is non-empty it will - include a trailing space - """ - result = "" - if c.invokeacl != AccessPrivilege.OPERATE: - result += "invoke: " + human_text_string(c.invokeacl) + """Generates the access string required for an event. If string is non-empty it will + include a trailing space + """ + result = "" + if c.invokeacl != AccessPrivilege.OPERATE: + result += "invoke: " + human_text_string(c.invokeacl) - if not result: - return "" - return f"access({result}) " + if not result: + return "" + return f"access({result}) " class IdlGenerator(CodeGenerator): @@ -122,7 +125,6 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): self.jinja_env.filters['event_access'] = event_access_string self.jinja_env.filters['command_access'] = command_access_string - def internal_render_all(self): """ Renders the output. @@ -136,4 +138,3 @@ def internal_render_all(self): 'idl': self.idl } ) - From cae70053970a63413950aa7c2f9827d1770c5644 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:16:53 -0400 Subject: [PATCH 09/36] Add descriptions to clusters --- scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index c44b907be7d07b..c6e6299505657e 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -34,6 +34,8 @@ // This IDL was auto-generated from a parsed data structure {% for cluster in idl.clusters -%} +{% if cluster.description %}/** {{cluster.description}} */ +{% endif -%} {{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { {%- for enum in cluster.enums %} From c0ec33881b0fae5284c9b0e1a41a28ff92428147 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:35:21 -0400 Subject: [PATCH 10/36] Attempt to fix up alignment of things --- .../matter_idl/generators/idl/MatterIdl.jinja | 59 +++++++++++-------- .../matter_idl/generators/idl/__init__.py | 4 ++ 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index c6e6299505657e..7272d098a08479 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -23,60 +23,71 @@ struct {{s.name}} {##} {%- if s.code %}= {{s.code}} {% endif -%} { - {%- for field in s.fields %} + {% for field in s.fields %} {{render_field(field)}} - {%- endfor %} + {% endfor %} } - {%- endmacro -%} // This IDL was auto-generated from a parsed data structure -{% for cluster in idl.clusters -%} +{% for cluster in idl.clusters %} + {% if cluster.description %}/** {{cluster.description}} */ -{% endif -%} -{{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { - {%- for enum in cluster.enums %} +{% endif %} +{{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { + {% for enum in cluster.enums %} enum {{enum.name}} : {{ enum.base_type}} { - {%- for entry in enum.entries %} + {% for entry in enum.entries %} {{entry.name}} = {{entry.code}}; - {%- endfor %} + {% endfor %} } - {%- endfor %} - {%- for bitmap in cluster.bitmaps %} + + {% endfor %} + + {% for bitmap in cluster.bitmaps %} bitmap {{bitmap.name}} : {{ bitmap.base_type}} { - {%- for entry in bitmap.entries %} + {% for entry in bitmap.entries %} {{entry.name}} = {{entry.code}}; - {%- endfor %} + {% endfor %} } - {%- endfor %} + {% endfor %} + {% for s in cluster.structs | rejectattr("tag") -%} {{render_struct(s)}} {% endfor %} - {%- for e in cluster.events %} + + {% for e in cluster.events %} {% if e.qualities %}{{e.qualities | idltxt}} {% endif -%} {{e.priority | idltxt}} event {{e | event_access}}{{e.name}} = {{e.code}} { - {%- for field in e.fields %} + {% for field in e.fields %} {{render_field(field)}} - {%- endfor %} + {% endfor %} } - {%- endfor %} - {%- for a in cluster.attributes %} + + {% endfor %} + + {% for a in cluster.attributes %} {{a.qualities | idltxt}}attribute {{render_field(a.definition)}} - {%- if loop.last %} + {% endfor %} + + {%- for s in cluster.structs | selectattr("tag") %} + {% if loop.first %} {% endif %} - {%- endfor %} - {%- for s in cluster.structs | selectattr("tag") -%} - {{render_struct(s)}} + {{render_struct(s)}} {% endfor %} - {% for c in cluster.commands -%} + + {%- for c in cluster.commands %} + {% if loop.first %} + + {% endif %} {{c.qualities | idltxt}}command {{c | command_access}}{{c.name}}({{c.input_param}}): {{c.output_param}} = {{c.code}}; {% endfor %} } diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 675a1133052b26..4437c1a563a9ba 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -125,6 +125,10 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): self.jinja_env.filters['event_access'] = event_access_string self.jinja_env.filters['command_access'] = command_access_string + # Easier whitespace management + self.jinja_env.trim_blocks = True + self.jinja_env.lstrip_blocks = True + def internal_render_all(self): """ Renders the output. From 41d555228331d1271eeba6b8fab28d67124e6cac Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:36:55 -0400 Subject: [PATCH 11/36] Alignment looks slightly better --- .../matter_idl/generators/idl/MatterIdl.jinja | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 7272d098a08479..fb6082ef8da0eb 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -38,16 +38,16 @@ {% endif %} {{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { - {% for enum in cluster.enums %} + {%- for enum in cluster.enums %} + enum {{enum.name}} : {{ enum.base_type}} { {% for entry in enum.entries %} {{entry.name}} = {{entry.code}}; {% endfor %} } - {% endfor %} - {% for bitmap in cluster.bitmaps %} + {%- for bitmap in cluster.bitmaps %} bitmap {{bitmap.name}} : {{ bitmap.base_type}} { {% for entry in bitmap.entries %} @@ -56,12 +56,12 @@ } {% endfor %} - {% for s in cluster.structs | rejectattr("tag") -%} + {%- for s in cluster.structs | rejectattr("tag") -%} {{render_struct(s)}} {% endfor %} - {% for e in cluster.events %} + {%- for e in cluster.events %} {% if e.qualities %}{{e.qualities | idltxt}} {% endif -%} {{e.priority | idltxt}} event {{e | event_access}}{{e.name}} = {{e.code}} { @@ -72,7 +72,10 @@ {% endfor %} - {% for a in cluster.attributes %} + {%- for a in cluster.attributes %} + {%- if loop.first %} + + {% endif %} {{a.qualities | idltxt}}attribute {{render_field(a.definition)}} {% endfor %} From f45f613adbf411c2b14439d9eb77e03696a3b7b3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:38:35 -0400 Subject: [PATCH 12/36] Better command separation --- .../py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index fb6082ef8da0eb..85fffd9a2a7e08 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -83,13 +83,17 @@ {% if loop.first %} {% endif %} - {{render_struct(s)}} + {%- if not loop.last %} + + + {% endif %} {% endfor %} {%- for c in cluster.commands %} {% if loop.first %} + {% endif %} {{c.qualities | idltxt}}command {{c | command_access}}{{c.name}}({{c.input_param}}): {{c.output_param}} = {{c.code}}; {% endfor %} From 5576d682c4c9aa043548c624bceeb32274f05a60 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:39:22 -0400 Subject: [PATCH 13/36] Align comments --- scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 85fffd9a2a7e08..7049ccb8f82a22 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -36,7 +36,6 @@ {% if cluster.description %}/** {{cluster.description}} */ {% endif %} - {{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { {%- for enum in cluster.enums %} From ce0f3432209e4a1781c90dd9ce1ec05028e3d341 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:41:38 -0400 Subject: [PATCH 14/36] Align and output descriptions including clusters --- .../py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 7049ccb8f82a22..3a4a04906a5ad9 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -33,7 +33,6 @@ // This IDL was auto-generated from a parsed data structure {% for cluster in idl.clusters %} - {% if cluster.description %}/** {{cluster.description}} */ {% endif %} {{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { @@ -93,6 +92,9 @@ {% if loop.first %} + {% endif %} + {% if c.description %} + /** {{c.description}} */ {% endif %} {{c.qualities | idltxt}}command {{c | command_access}}{{c.name}}({{c.input_param}}): {{c.output_param}} = {{c.code}}; {% endfor %} From 84bcad2925186dd6905860ad8304e090c6cfe562 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 11:58:04 -0400 Subject: [PATCH 15/36] More work regarding loop structures --- .../matter_idl/generators/idl/MatterIdl.jinja | 19 +++++++++++----- .../matter_idl/generators/idl/__init__.py | 22 +++++++++++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 3a4a04906a5ad9..75f56af41b06d7 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -21,7 +21,7 @@ {%- if s.tag %}{{s.tag | idltxt}} {% endif -%} {% if s.qualities %}{{s.qualities | idltxt}} {% endif -%} struct {{s.name}} {##} - {%- if s.code %}= {{s.code}} {% endif -%} + {%- if s.code is not none %}= {{s.code}} {% endif -%} { {% for field in s.fields %} {{render_field(field)}} @@ -54,14 +54,21 @@ } {% endfor %} - {%- for s in cluster.structs | rejectattr("tag") -%} + {%- for s in cluster.structs | rejectattr("tag") %} + {% if loop.first %} + + {% endif %} {{render_struct(s)}} + {% if not loop.last %} + {% endif %} {% endfor %} {%- for e in cluster.events %} + {% if loop.first %} - {% if e.qualities %}{{e.qualities | idltxt}} {% endif -%} + {% endif %} + {##} {##}{% if e.qualities %}{{e.qualities | idltxt}} {% endif -%} {{e.priority | idltxt}} event {{e | event_access}}{{e.name}} = {{e.code}} { {% for field in e.fields %} {{render_field(field)}} @@ -74,7 +81,7 @@ {%- if loop.first %} {% endif %} - {{a.qualities | idltxt}}attribute {{render_field(a.definition)}} + {{a.qualities | idltxt}}attribute {{a | attribute_access}}{{render_field(a.definition)}} {% endfor %} {%- for s in cluster.structs | selectattr("tag") %} @@ -96,7 +103,9 @@ {% if c.description %} /** {{c.description}} */ {% endif %} - {{c.qualities | idltxt}}command {{c | command_access}}{{c.name}}({{c.input_param}}): {{c.output_param}} = {{c.code}}; + {{c.qualities | idltxt}}command {{c | command_access}}{{c.name}}( + {%- if c.input_param %}{{c.input_param}}{% endif -%} + ): {{c.output_param}} = {{c.code}}; {% endfor %} } diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 4437c1a563a9ba..fb70ee26ec6ebf 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -17,7 +17,7 @@ from matter_idl.generators import CodeGenerator, GeneratorStorage from matter_idl.matter_idl_types import (AccessPrivilege, AttributeQuality, Cluster, ClusterSide, Command, CommandQuality, Event, - EventPriority, EventQuality, FieldQuality, Idl, Struct, StructQuality, StructTag) + EventPriority, EventQuality, FieldQuality, Idl, Struct, StructQuality, StructTag, Attribute) def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality) -> str: @@ -101,7 +101,7 @@ def event_access_string(e: Event) -> str: def command_access_string(c: Command) -> str: - """Generates the access string required for an event. If string is non-empty it will + """Generates the access string required for a command. If string is non-empty it will include a trailing space """ result = "" @@ -112,6 +112,23 @@ def command_access_string(c: Command) -> str: return "" return f"access({result}) " +def attribute_access_string(a: Attribute) -> str: + """Generates the access string required for a struct. If string is non-empty it will + include a trailing space + """ + result = [] + + if a.readacl != AccessPrivilege.VIEW: + result.append("read: " + human_text_string(a.readacl)) + + if a.writeacl != AccessPrivilege.OPERATE: + result.append("write: " + human_text_string(a.writeacl)) + + if not result: + return "" + + return f"access({', '.join(result)}) " + class IdlGenerator(CodeGenerator): """ @@ -124,6 +141,7 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): self.jinja_env.filters['idltxt'] = human_text_string self.jinja_env.filters['event_access'] = event_access_string self.jinja_env.filters['command_access'] = command_access_string + self.jinja_env.filters['attribute_access'] = attribute_access_string # Easier whitespace management self.jinja_env.trim_blocks = True From d25a38a8b141963391421ac2e37ea78e92556157 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:02:03 -0400 Subject: [PATCH 16/36] Apply hex formatting to bitmaps. output now seems identical except one whitespace change --- .../py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 75f56af41b06d7..733fe90fe3f055 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -49,7 +49,7 @@ bitmap {{bitmap.name}} : {{ bitmap.base_type}} { {% for entry in bitmap.entries %} - {{entry.name}} = {{entry.code}}; + {{entry.name}} = 0x{{"%X" | format(entry.code)}}; {% endfor %} } {% endfor %} @@ -78,7 +78,7 @@ {% endfor %} {%- for a in cluster.attributes %} - {%- if loop.first %} + {% if loop.first %} {% endif %} {{a.qualities | idltxt}}attribute {{a | attribute_access}}{{render_field(a.definition)}} From 9c94d69d2de8c07cdffed612bedc9d146ac082ee Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:16:16 -0400 Subject: [PATCH 17/36] Identical output for now --- .../matter_idl/generators/idl/MatterIdl.jinja | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 733fe90fe3f055..b8ae56935abaab 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -74,7 +74,9 @@ {{render_field(field)}} {% endfor %} } + {% if not loop.last %} + {% endif %} {% endfor %} {%- for a in cluster.attributes %} @@ -85,20 +87,13 @@ {% endfor %} {%- for s in cluster.structs | selectattr("tag") %} - {% if loop.first %} - {% endif %} {{render_struct(s)}} - {%- if not loop.last %} - - - {% endif %} {% endfor %} {%- for c in cluster.commands %} {% if loop.first %} - {% endif %} {% if c.description %} /** {{c.description}} */ From 66089f4de445dedca6f75848ac725adf14e58d71 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:20:47 -0400 Subject: [PATCH 18/36] Support API maturity. Notice that doccomments are lost on maturity :( --- .../matter_idl/generators/idl/MatterIdl.jinja | 2 +- .../matter_idl/generators/idl/__init__.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index b8ae56935abaab..65ef6e99277f5f 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -35,7 +35,7 @@ {% for cluster in idl.clusters %} {% if cluster.description %}/** {{cluster.description}} */ {% endif %} -{{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { +{{cluster.api_maturity | idltxt}}{{cluster.side | idltxt}} cluster {{cluster.name}} = {{cluster.code}} { {%- for enum in cluster.enums %} enum {{enum.name}} : {{ enum.base_type}} { diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index fb70ee26ec6ebf..8f58c523c66dc2 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -17,10 +17,10 @@ from matter_idl.generators import CodeGenerator, GeneratorStorage from matter_idl.matter_idl_types import (AccessPrivilege, AttributeQuality, Cluster, ClusterSide, Command, CommandQuality, Event, - EventPriority, EventQuality, FieldQuality, Idl, Struct, StructQuality, StructTag, Attribute) + EventPriority, EventQuality, FieldQuality, Idl, Struct, StructQuality, StructTag, Attribute, ApiMaturity) -def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality) -> str: +def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality | ApiMaturity) -> str: if type(value) is ClusterSide: if value == ClusterSide.CLIENT: return "client" @@ -82,6 +82,15 @@ def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPrio if CommandQuality.FABRIC_SCOPED in value: result += "fabric " return result + elif type(value) is ApiMaturity: + if value == ApiMaturity.STABLE: + return "" + if value == ApiMaturity.PROVISIONAL: + return "provisional " + if value == ApiMaturity.INTERNAL: + return "internal " + if value == ApiMaturity.DEPRECATED: + return "deprecated " # wrong value in general return "Unknown/unsupported: %r" % value From 1708396d59a1c00ec886e5dcae97e016092062e0 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:25:34 -0400 Subject: [PATCH 19/36] Fix doxygen parsing for api maturity at the cluster level --- scripts/py_matter_idl/matter_idl/matter_idl_parser.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/py_matter_idl/matter_idl/matter_idl_parser.py b/scripts/py_matter_idl/matter_idl/matter_idl_parser.py index 0d1bfb8d869e49..ca606cae9fdd66 100755 --- a/scripts/py_matter_idl/matter_idl/matter_idl_parser.py +++ b/scripts/py_matter_idl/matter_idl/matter_idl_parser.py @@ -46,6 +46,14 @@ def appply_to_idl(self, idl: Idl, content: str): while content[actual_pos] in ' \t\n\r': actual_pos += 1 + # Allow to skip api maturity flags + for maturity in ["provisional", "internal", "stable", "deprecated"]: + if content[actual_pos:].startswith(maturity): + actual_pos += len(maturity) + + while content[actual_pos] in ' \t\n\r': + actual_pos += 1 + # A doc comment will apply to any supported element assuming it immediately # preceeds id (skipping whitespace) for item in self.supported_types(idl): From 09c5adb362c4c70c0daf2977fb968878b60ece24 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:26:05 -0400 Subject: [PATCH 20/36] Restyle --- scripts/py_matter_idl/matter_idl/generators/idl/__init__.py | 6 ++++-- scripts/py_matter_idl/matter_idl/matter_idl_parser.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 8f58c523c66dc2..0ddce9ec4fcef2 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -16,8 +16,9 @@ from typing import List from matter_idl.generators import CodeGenerator, GeneratorStorage -from matter_idl.matter_idl_types import (AccessPrivilege, AttributeQuality, Cluster, ClusterSide, Command, CommandQuality, Event, - EventPriority, EventQuality, FieldQuality, Idl, Struct, StructQuality, StructTag, Attribute, ApiMaturity) +from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, Cluster, ClusterSide, Command, + CommandQuality, Event, EventPriority, EventQuality, FieldQuality, Idl, Struct, + StructQuality, StructTag) def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality | ApiMaturity) -> str: @@ -121,6 +122,7 @@ def command_access_string(c: Command) -> str: return "" return f"access({result}) " + def attribute_access_string(a: Attribute) -> str: """Generates the access string required for a struct. If string is non-empty it will include a trailing space diff --git a/scripts/py_matter_idl/matter_idl/matter_idl_parser.py b/scripts/py_matter_idl/matter_idl/matter_idl_parser.py index ca606cae9fdd66..711e2535887b5e 100755 --- a/scripts/py_matter_idl/matter_idl/matter_idl_parser.py +++ b/scripts/py_matter_idl/matter_idl/matter_idl_parser.py @@ -48,8 +48,8 @@ def appply_to_idl(self, idl: Idl, content: str): # Allow to skip api maturity flags for maturity in ["provisional", "internal", "stable", "deprecated"]: - if content[actual_pos:].startswith(maturity): - actual_pos += len(maturity) + if content[actual_pos:].startswith(maturity): + actual_pos += len(maturity) while content[actual_pos] in ' \t\n\r': actual_pos += 1 From 75e16b384d2c284e766eee7ca314820bd315ac9d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:45:01 -0400 Subject: [PATCH 21/36] Support endpoints, although that is not 1:1 as hex encoding and ordering for events is lost --- .../matter_idl/generators/idl/MatterIdl.jinja | 40 +++++++++++++++++-- .../matter_idl/generators/idl/__init__.py | 28 ++++++++++++- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 65ef6e99277f5f..55568d22e7b6fb 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -107,10 +107,44 @@ {% endfor %} {%- if idl.endpoints %} -{% for endpoint in idl.endpoints %} - +{%- for endpoint in idl.endpoints %} endpoint {{endpoint.number}} { - // FIXME: report endpoint + {% for t in endpoint.device_types %} + device type {{t.name}} = {{t.code}}, version {{t.version}}; + {% endfor%} + + {%-for b in endpoint.client_bindings %} + {% if loop.first %} + + {% endif %} + binding cluster {{b}}; + {% endfor %} + + {%-for c in endpoint.server_clusters %} + + server cluster {{c.name}} { + {% for e in c.events_emitted %} + emits event {{e}}; + {% if loop.last %} + + {% endif %} + {% endfor %} + {% for a in c.attributes %} + {{"%-8s" | format(a.storage|idltxt) }} attribute {{a.name}} + {%- if a.default is not none %} default = {{a.default}} {%- endif %}; + {% endfor %} + {% for cmd in c.commands %} + {% if loop.first %} + + {% endif %} + handle command {{cmd.name}}; + {% endfor %} + } + {% endfor %} + } +{% if not loop.last %} + +{% endif %} {% endfor %} {% endif %} diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 0ddce9ec4fcef2..06a649d1e239a9 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -18,10 +18,10 @@ from matter_idl.generators import CodeGenerator, GeneratorStorage from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, Cluster, ClusterSide, Command, CommandQuality, Event, EventPriority, EventQuality, FieldQuality, Idl, Struct, - StructQuality, StructTag) + StructQuality, StructTag, AttributeStorage) -def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality | ApiMaturity) -> str: +def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality | ApiMaturity | AttributeStorage) -> str: if type(value) is ClusterSide: if value == ClusterSide.CLIENT: return "client" @@ -92,6 +92,13 @@ def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPrio return "internal " if value == ApiMaturity.DEPRECATED: return "deprecated " + elif type(value) is AttributeStorage: + if value == AttributeStorage.RAM: + return "ram" + if value == AttributeStorage.PERSIST: + return "persist" + if value == AttributeStorage.CALLBACK: + return "callback" # wrong value in general return "Unknown/unsupported: %r" % value @@ -141,6 +148,22 @@ def attribute_access_string(a: Attribute) -> str: return f"access({', '.join(result)}) " +def render_default(value: str|int|bool) -> str: + """ + Renders a idl-style default. + + Generally quotes strings and handles bools + """ + if type(value) is str: + return f'"{value}"' + elif type(value) is bool: + if value: + return "true" + else: + return "false" + return str(value) + + class IdlGenerator(CodeGenerator): """ Generation .matter idl files for a given IDL @@ -153,6 +176,7 @@ def __init__(self, storage: GeneratorStorage, idl: Idl, **kargs): self.jinja_env.filters['event_access'] = event_access_string self.jinja_env.filters['command_access'] = command_access_string self.jinja_env.filters['attribute_access'] = attribute_access_string + self.jinja_env.filters['render_default'] = render_default # Easier whitespace management self.jinja_env.trim_blocks = True From 97532b8a7cd88bc3d663e8cea891b1d3bb70e024 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:45:36 -0400 Subject: [PATCH 22/36] Restyle --- .../py_matter_idl/matter_idl/generators/idl/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 06a649d1e239a9..74ab01a5e2f4ea 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -16,9 +16,9 @@ from typing import List from matter_idl.generators import CodeGenerator, GeneratorStorage -from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, Cluster, ClusterSide, Command, - CommandQuality, Event, EventPriority, EventQuality, FieldQuality, Idl, Struct, - StructQuality, StructTag, AttributeStorage) +from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, AttributeStorage, Cluster, + ClusterSide, Command, CommandQuality, Event, EventPriority, EventQuality, FieldQuality, + Idl, Struct, StructQuality, StructTag) def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality | ApiMaturity | AttributeStorage) -> str: @@ -148,7 +148,7 @@ def attribute_access_string(a: Attribute) -> str: return f"access({', '.join(result)}) " -def render_default(value: str|int|bool) -> str: +def render_default(value: str | int | bool) -> str: """ Renders a idl-style default. From 14c5920a9d6b0c8115e0e902e5fb7fcadeaa389c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:46:54 -0400 Subject: [PATCH 23/36] Add todo note that default value does not string escaping --- scripts/py_matter_idl/matter_idl/generators/idl/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 74ab01a5e2f4ea..c21fc4b038dc93 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -155,6 +155,9 @@ def render_default(value: str | int | bool) -> str: Generally quotes strings and handles bools """ if type(value) is str: + # TODO: technically this should support escaping for quotes + # however currently we never needed this. Escaping can be + # added once we use this info return f'"{value}"' elif type(value) is bool: if value: From 01d3b77b5498acf919c2f9a974c257701f4f2f15 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:58:10 -0400 Subject: [PATCH 24/36] Default rendering and add to files --- scripts/py_matter_idl/files.gni | 2 ++ scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/py_matter_idl/files.gni b/scripts/py_matter_idl/files.gni index 84680a8863d42f..8842a66f4fb432 100644 --- a/scripts/py_matter_idl/files.gni +++ b/scripts/py_matter_idl/files.gni @@ -7,6 +7,7 @@ matter_idl_generator_templates = [ "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/PluginApplicationCallbacksHeader.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_cpp.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/TLVMetaData_h.jinja", + "${chip_root}/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/CHIPCallbackTypes.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersCpp.jinja", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/ChipClustersRead.jinja", @@ -30,6 +31,7 @@ matter_idl_generator_sources = [ "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/registry.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/types.py", + "${chip_root}/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/lint/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/lint/lint_rules_parser.py", "${chip_root}/scripts/py_matter_idl/matter_idl/lint/types.py", diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja index 55568d22e7b6fb..91e978a44c7b07 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja +++ b/scripts/py_matter_idl/matter_idl/generators/idl/MatterIdl.jinja @@ -131,7 +131,7 @@ endpoint {{endpoint.number}} { {% endfor %} {% for a in c.attributes %} {{"%-8s" | format(a.storage|idltxt) }} attribute {{a.name}} - {%- if a.default is not none %} default = {{a.default}} {%- endif %}; + {%- if a.default is not none %} default = {{a.default|render_default}} {%- endif %}; {% endfor %} {% for cmd in c.commands %} {% if loop.first %} From 0b8e5a829569570d576fd954df59e366fb202a1a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 12:59:54 -0400 Subject: [PATCH 25/36] More updates on file dependencies --- scripts/py_matter_idl/setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/py_matter_idl/setup.cfg b/scripts/py_matter_idl/setup.cfg index 57f93c3dd9aa82..714b42a46e614e 100644 --- a/scripts/py_matter_idl/setup.cfg +++ b/scripts/py_matter_idl/setup.cfg @@ -34,6 +34,7 @@ matter_idl = generators/cpp/application/PluginApplicationCallbacksHeader.jinja generators/cpp/tlvmeta/TLVMetaData_cpp.jinja generators/cpp/tlvmeta/TLVMetaData_h.jinja + generators/idl/MatterIdl.jinja generators/java/CHIPCallbackTypes.jinja generators/java/ChipClustersCpp.jinja generators/java/ChipClustersRead.jinja From 94b6ec1e9e54e0510deaf2c12b4fba9f00d5c86f Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 13:23:03 -0400 Subject: [PATCH 26/36] Unit test IDL generator --- .../matter_idl/test_idl_generator.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 scripts/py_matter_idl/matter_idl/test_idl_generator.py diff --git a/scripts/py_matter_idl/matter_idl/test_idl_generator.py b/scripts/py_matter_idl/matter_idl/test_idl_generator.py new file mode 100755 index 00000000000000..05e1efcdccfb45 --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/test_idl_generator.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest +from dataclasses import dataclass, field +from typing import List + +import yaml + +try: + from matter_idl.matter_idl_parser import CreateParser +except ImportError: + + sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..'))) + from matter_idl.matter_idl_parser import CreateParser + +from matter_idl.generators import GeneratorStorage +from matter_idl.generators.idl import IdlGenerator +from matter_idl.matter_idl_types import Idl + + +class TestCaseStorage(GeneratorStorage): + def __init__(self): + super().__init__() + self.content: Optional[str] = None + + def get_existing_data(self, relative_path: str): + # Force re-generation each time + return None + + def write_new_data(self, relative_path: str, content: str): + if self.content: + raise Exception("Unexpected extra data: single file generation expected") + self.content = content + +def ReadMatterIdl(repo_path: str) -> Idl: + path = os.path.join(os.path.dirname(__file__), "../../..", repo_path) + with open(path, "rt") as stream: + return stream.read() + +def ParseMatterIdl(repo_path: str, skip_meta: bool) -> Idl: + return CreateParser(skip_meta=skip_meta).parse(ReadMatterIdl(repo_path)) + +def RenderAsIdlTxt(idl: Idl) -> str: + storage = TestCaseStorage() + IdlGenerator(storage=storage, idl=idl).render(dry_run=False) + return storage.content + +def SkipLeadingComments(txt: str) -> str: + """Skips leading lines starting with // in a file. """ + lines = txt.split("\n") + idx = 0 + while lines[idx].startswith("//") or not lines[idx]: + idx = idx + 1 + return "\n".join(lines[idx:]) + + +class TestIdlRendering(unittest.TestCase): + def test_client_clusters(self): + # IDL renderer was updated to have IDENTICAL output for client side + # cluster rendering, so this diff will be verbatim + # + # Comparison made text-mode so that meta-data is read and doc-comments are + # available + + path = "src/controller/data_model/controller-clusters.matter" + + # Files MUST be identical except the header comments which are different + original = SkipLeadingComments(ReadMatterIdl(path)) + generated = SkipLeadingComments(RenderAsIdlTxt(ParseMatterIdl(path, skip_meta=False))) + + self.assertEqual(original, generated) + + def test_app_rendering(self): + # When endpoints are involved, default value formatting is lost + # (e.g. "0x0000" becomes "0") and ordering of emitted events is not preserved + # because the events are a + + # as such, this test validates that parsing + generating + re-parsing results + # in the same data being parsed + test_paths = [ + "examples/lock-app/lock-common/lock-app.matter", + "examples/lighting-app/lighting-common/lighting-app.matter", + "examples/all-clusters-app/all-clusters-common/all-clusters-app.matter", + "examples/thermostat/thermostat-common/thermostat.matter", + ] + + for path in test_paths: + idl = ParseMatterIdl(path, skip_meta=True) + txt = RenderAsIdlTxt(idl) + idl2 = CreateParser(skip_meta=True).parse(txt) + + # checks that data types and content is the same + self.assertEqual(idl, idl2) + + +if __name__ == '__main__': + unittest.main() From 792f1d04fdb4e2f06084a712b5f6f44df45b3076 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 13:24:03 -0400 Subject: [PATCH 27/36] Add the IDL unit test as a standard unit test --- scripts/py_matter_idl/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/py_matter_idl/BUILD.gn b/scripts/py_matter_idl/BUILD.gn index 81f5be3be554b4..48fc68aea40268 100644 --- a/scripts/py_matter_idl/BUILD.gn +++ b/scripts/py_matter_idl/BUILD.gn @@ -62,6 +62,7 @@ pw_python_package("matter_idl") { "matter_idl/test_backwards_compatibility.py", "matter_idl/test_matter_idl_parser.py", "matter_idl/test_generators.py", + "matter_idl/test_idl_generator.py", "matter_idl/test_xml_parser.py", ] From 00f1efae4d2e68d29b8fc644d32996f2a67c63f9 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 13:39:30 -0400 Subject: [PATCH 28/36] Update for python compatibility --- scripts/py_matter_idl/matter_idl/generators/idl/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index c21fc4b038dc93..15cf8c5113b405 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from typing import List +from typing import List, Union from matter_idl.generators import CodeGenerator, GeneratorStorage from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, AttributeStorage, Cluster, @@ -21,7 +21,7 @@ Idl, Struct, StructQuality, StructTag) -def human_text_string(value: ClusterSide | StructTag | StructQuality | EventPriority | EventQuality | AccessPrivilege | AttributeQuality | CommandQuality | ApiMaturity | AttributeStorage) -> str: +def human_text_string(value: Union[ClusterSide, StructTag, StructQuality, EventPriority, EventQuality, AccessPrivilege, AttributeQuality, CommandQuality, ApiMaturity, AttributeStorage]) -> str: if type(value) is ClusterSide: if value == ClusterSide.CLIENT: return "client" @@ -148,7 +148,7 @@ def attribute_access_string(a: Attribute) -> str: return f"access({', '.join(result)}) " -def render_default(value: str | int | bool) -> str: +def render_default(value: Union[str, int, bool]) -> str: """ Renders a idl-style default. From 7b4ed272bff468ddef244dfbdfa72b7cde026fd2 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 13:44:12 -0400 Subject: [PATCH 29/36] Fix unit testing of builds when GSDK root is defined --- scripts/build/test.py | 1 + .../testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt | 2 +- third_party/bouffalolab/repo | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build/test.py b/scripts/build/test.py index c89a4d98109357..f684ace074d5ca 100644 --- a/scripts/build/test.py +++ b/scripts/build/test.py @@ -48,6 +48,7 @@ def build_actual_output(root: str, out: str, args: List[str]) -> List[str]: 'IMX_SDK_ROOT': 'IMX_SDK_ROOT', 'TI_SYSCONFIG_ROOT': 'TEST_TI_SYSCONFIG_ROOT', 'JAVA_PATH': 'TEST_JAVA_PATH', + 'GSDK_ROOT': 'TEST_GSDK_ROOT', }) retval = subprocess.run([ diff --git a/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt b/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt index 309cdd49b9f256..b96ccbc89ea97e 100644 --- a/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt +++ b/scripts/build/testdata/dry_run_efr32-brd4161a-light-rpc-no-version.txt @@ -2,7 +2,7 @@ cd "{root}" # Generating efr32-brd4161a-light-rpc-no-version -gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/lighting-app/silabs '--args=silabs_board="BRD4161A" is_debug=false import("//with_pw_rpc.gni")' {out}/efr32-brd4161a-light-rpc-no-version +gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/lighting-app/silabs '--args=silabs_board="BRD4161A" is_debug=false import("//with_pw_rpc.gni") efr32_sdk_root="TEST_GSDK_ROOT" openthread_root="TEST_GSDK_ROOT/util/third_party/openthread"' {out}/efr32-brd4161a-light-rpc-no-version # Building efr32-brd4161a-light-rpc-no-version ninja -C {out}/efr32-brd4161a-light-rpc-no-version diff --git a/third_party/bouffalolab/repo b/third_party/bouffalolab/repo index f65861db5bee5b..690b348635806f 160000 --- a/third_party/bouffalolab/repo +++ b/third_party/bouffalolab/repo @@ -1 +1 @@ -Subproject commit f65861db5bee5b24aeeae5b6039cefabcdc57294 +Subproject commit 690b348635806f312b61ddcc5b2c566daafe833e From 49481110a6d1f321f31880549e181e2d29e4d789 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 13:53:30 -0400 Subject: [PATCH 30/36] Added a readme file --- .../matter_idl/generators/idl/README.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 scripts/py_matter_idl/matter_idl/generators/idl/README.md diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/README.md b/scripts/py_matter_idl/matter_idl/generators/idl/README.md new file mode 100644 index 00000000000000..96355cca0d4426 --- /dev/null +++ b/scripts/py_matter_idl/matter_idl/generators/idl/README.md @@ -0,0 +1,20 @@ +## Generator description + +Generates a structured `Idl` data type into a human-readable +text format (`.matter` file). + +It is useful for tools that ingest non-idl data but convert into +idl data (e.g. zapxml or CSA data model XML data.) + +### Usage + +A no-op usage can be: + +``` +./scripts/codegen.py -g idl --output-dir out/idlgen examples/all-clusters-app/all-clusters-common/all-clusters-app.matter +``` + +which would re-generate the entire `all-clusters-app.matter` into `out/idlgen/idl.matter` + +This generation is useful for testing/validating that both parsing and generation works. Actual +usage of this generator would be inside XML tools. From 5b33085a25ccdda1c727e8f483bd8602c200a47d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 13:53:59 -0400 Subject: [PATCH 31/36] Restyle --- scripts/py_matter_idl/files.gni | 2 +- .../matter_idl/generators/idl/README.md | 15 ++++++++------- .../matter_idl/test_idl_generator.py | 6 +++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/py_matter_idl/files.gni b/scripts/py_matter_idl/files.gni index 8842a66f4fb432..cb54b253ef0615 100644 --- a/scripts/py_matter_idl/files.gni +++ b/scripts/py_matter_idl/files.gni @@ -28,10 +28,10 @@ matter_idl_generator_sources = [ "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/tlvmeta/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/filters.py", + "${chip_root}/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/java/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/registry.py", "${chip_root}/scripts/py_matter_idl/matter_idl/generators/types.py", - "${chip_root}/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/lint/__init__.py", "${chip_root}/scripts/py_matter_idl/matter_idl/lint/lint_rules_parser.py", "${chip_root}/scripts/py_matter_idl/matter_idl/lint/types.py", diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/README.md b/scripts/py_matter_idl/matter_idl/generators/idl/README.md index 96355cca0d4426..706c629afba75c 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/README.md +++ b/scripts/py_matter_idl/matter_idl/generators/idl/README.md @@ -1,10 +1,10 @@ ## Generator description -Generates a structured `Idl` data type into a human-readable -text format (`.matter` file). +Generates a structured `Idl` data type into a human-readable text format +(`.matter` file). -It is useful for tools that ingest non-idl data but convert into -idl data (e.g. zapxml or CSA data model XML data.) +It is useful for tools that ingest non-idl data but convert into idl data (e.g. +zapxml or CSA data model XML data.) ### Usage @@ -14,7 +14,8 @@ A no-op usage can be: ./scripts/codegen.py -g idl --output-dir out/idlgen examples/all-clusters-app/all-clusters-common/all-clusters-app.matter ``` -which would re-generate the entire `all-clusters-app.matter` into `out/idlgen/idl.matter` +which would re-generate the entire `all-clusters-app.matter` into +`out/idlgen/idl.matter` -This generation is useful for testing/validating that both parsing and generation works. Actual -usage of this generator would be inside XML tools. +This generation is useful for testing/validating that both parsing and +generation works. Actual usage of this generator would be inside XML tools. diff --git a/scripts/py_matter_idl/matter_idl/test_idl_generator.py b/scripts/py_matter_idl/matter_idl/test_idl_generator.py index 05e1efcdccfb45..cc8e26a56bac62 100755 --- a/scripts/py_matter_idl/matter_idl/test_idl_generator.py +++ b/scripts/py_matter_idl/matter_idl/test_idl_generator.py @@ -49,19 +49,23 @@ def write_new_data(self, relative_path: str, content: str): raise Exception("Unexpected extra data: single file generation expected") self.content = content + def ReadMatterIdl(repo_path: str) -> Idl: path = os.path.join(os.path.dirname(__file__), "../../..", repo_path) with open(path, "rt") as stream: return stream.read() + def ParseMatterIdl(repo_path: str, skip_meta: bool) -> Idl: return CreateParser(skip_meta=skip_meta).parse(ReadMatterIdl(repo_path)) + def RenderAsIdlTxt(idl: Idl) -> str: storage = TestCaseStorage() IdlGenerator(storage=storage, idl=idl).render(dry_run=False) return storage.content + def SkipLeadingComments(txt: str) -> str: """Skips leading lines starting with // in a file. """ lines = txt.split("\n") @@ -82,7 +86,7 @@ def test_client_clusters(self): path = "src/controller/data_model/controller-clusters.matter" # Files MUST be identical except the header comments which are different - original = SkipLeadingComments(ReadMatterIdl(path)) + original = SkipLeadingComments(ReadMatterIdl(path)) generated = SkipLeadingComments(RenderAsIdlTxt(ParseMatterIdl(path, skip_meta=False))) self.assertEqual(original, generated) From 044376675ac8a56cb3eb87cf44f8d10b52785002 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 13:57:58 -0400 Subject: [PATCH 32/36] Make xml parser use the idl codegen --- .../matter_idl/generators/idl/README.md | 12 +++++++++ .../py_matter_idl/matter_idl/xml_parser.py | 25 ++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/README.md b/scripts/py_matter_idl/matter_idl/generators/idl/README.md index 706c629afba75c..ff3faff4aa1655 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/README.md +++ b/scripts/py_matter_idl/matter_idl/generators/idl/README.md @@ -19,3 +19,15 @@ which would re-generate the entire `all-clusters-app.matter` into This generation is useful for testing/validating that both parsing and generation works. Actual usage of this generator would be inside XML tools. + +### Within XML parsers + +A XML parser will use this code generator to output a human readable view of +the parsed data: + +``` +./scripts/py_matter_idl/matter_idl/xml_parser.py \ + ./src/app/zap-templates/zcl/data-model/chip/onoff-cluster.xml \ + ./src/app/zap-templates/zcl/data-model/chip/global-attributes.xm +``` + diff --git a/scripts/py_matter_idl/matter_idl/xml_parser.py b/scripts/py_matter_idl/matter_idl/xml_parser.py index 5a07490e9c3b68..e2b18c2c74da56 100755 --- a/scripts/py_matter_idl/matter_idl/xml_parser.py +++ b/scripts/py_matter_idl/matter_idl/xml_parser.py @@ -28,11 +28,23 @@ if __name__ == '__main__': - # This Parser is generally not intended to be run as a stand-alone binary. - # The ability to run is for debug and to print out the parsed AST. - import pprint - import click + from matter_idl.generators import GeneratorStorage + from matter_idl.generators.idl import IdlGenerator + + class InMemoryStorage(GeneratorStorage): + def __init__(self): + super().__init__() + self.content: Optional[str] = None + + def get_existing_data(self, relative_path: str): + # Force re-generation each time + return None + + def write_new_data(self, relative_path: str, content: str): + if self.content: + raise Exception("Unexpected extra data: single file generation expected") + self.content = content # Supported log levels, mapping string values required for argument # parsing into logging constants @@ -70,7 +82,8 @@ def main(log_level, no_print, filenames): logging.info("Parse completed") if not no_print: - print("Data:") - pprint.pp(data) + storage = InMemoryStorage() + IdlGenerator(storage=storage, idl=data).render(dry_run=False) + print(storage.content) main(auto_envvar_prefix='CHIP') From 640eeb28280721fb5298ac4c6817953607b8f56d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 14:00:14 -0400 Subject: [PATCH 33/36] Restyle --- .../matter_idl/generators/idl/README.md | 5 ++--- .../py_matter_idl/matter_idl/xml_parser.py | 20 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/README.md b/scripts/py_matter_idl/matter_idl/generators/idl/README.md index ff3faff4aa1655..8f53aa9ca7f220 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/README.md +++ b/scripts/py_matter_idl/matter_idl/generators/idl/README.md @@ -22,12 +22,11 @@ generation works. Actual usage of this generator would be inside XML tools. ### Within XML parsers -A XML parser will use this code generator to output a human readable view of -the parsed data: +A XML parser will use this code generator to output a human readable view of the +parsed data: ``` ./scripts/py_matter_idl/matter_idl/xml_parser.py \ ./src/app/zap-templates/zcl/data-model/chip/onoff-cluster.xml \ ./src/app/zap-templates/zcl/data-model/chip/global-attributes.xm ``` - diff --git a/scripts/py_matter_idl/matter_idl/xml_parser.py b/scripts/py_matter_idl/matter_idl/xml_parser.py index e2b18c2c74da56..6fcd987d78661f 100755 --- a/scripts/py_matter_idl/matter_idl/xml_parser.py +++ b/scripts/py_matter_idl/matter_idl/xml_parser.py @@ -33,18 +33,18 @@ from matter_idl.generators.idl import IdlGenerator class InMemoryStorage(GeneratorStorage): - def __init__(self): - super().__init__() - self.content: Optional[str] = None + def __init__(self): + super().__init__() + self.content: Optional[str] = None - def get_existing_data(self, relative_path: str): - # Force re-generation each time - return None + def get_existing_data(self, relative_path: str): + # Force re-generation each time + return None - def write_new_data(self, relative_path: str, content: str): - if self.content: - raise Exception("Unexpected extra data: single file generation expected") - self.content = content + def write_new_data(self, relative_path: str, content: str): + if self.content: + raise Exception("Unexpected extra data: single file generation expected") + self.content = content # Supported log levels, mapping string values required for argument # parsing into logging constants From cbdf613abb8affe74ee471715aae91a7cbb8a415 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 14:02:33 -0400 Subject: [PATCH 34/36] look to fix misspell warnings --- scripts/py_matter_idl/matter_idl/generators/idl/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/README.md b/scripts/py_matter_idl/matter_idl/generators/idl/README.md index 8f53aa9ca7f220..81e4b92c05e535 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/README.md +++ b/scripts/py_matter_idl/matter_idl/generators/idl/README.md @@ -4,7 +4,7 @@ Generates a structured `Idl` data type into a human-readable text format (`.matter` file). It is useful for tools that ingest non-idl data but convert into idl data (e.g. -zapxml or CSA data model XML data.) +`zapxml` or CSA data model XML data.) ### Usage @@ -20,7 +20,7 @@ which would re-generate the entire `all-clusters-app.matter` into This generation is useful for testing/validating that both parsing and generation works. Actual usage of this generator would be inside XML tools. -### Within XML parsers +### Within XML parsing A XML parser will use this code generator to output a human readable view of the parsed data: From b6de4df87097de7a96b119965acd8664a3a12d77 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 14:10:05 -0400 Subject: [PATCH 35/36] Undo repo update --- third_party/bouffalolab/repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/bouffalolab/repo b/third_party/bouffalolab/repo index 690b348635806f..f65861db5bee5b 160000 --- a/third_party/bouffalolab/repo +++ b/third_party/bouffalolab/repo @@ -1 +1 @@ -Subproject commit 690b348635806f312b61ddcc5b2c566daafe833e +Subproject commit f65861db5bee5b24aeeae5b6039cefabcdc57294 From 9a60d036ac70851514b707994db2db0dcb20d7a3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 19 Oct 2023 16:19:10 -0400 Subject: [PATCH 36/36] Fix linter errors --- .../py_matter_idl/matter_idl/generators/idl/__init__.py | 8 ++++---- scripts/py_matter_idl/matter_idl/test_idl_generator.py | 5 +---- scripts/py_matter_idl/matter_idl/xml_parser.py | 1 + 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py index 15cf8c5113b405..f53e35a7aba8e0 100644 --- a/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py +++ b/scripts/py_matter_idl/matter_idl/generators/idl/__init__.py @@ -13,12 +13,12 @@ # limitations under the License. import os -from typing import List, Union +from typing import Union from matter_idl.generators import CodeGenerator, GeneratorStorage -from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, AttributeStorage, Cluster, - ClusterSide, Command, CommandQuality, Event, EventPriority, EventQuality, FieldQuality, - Idl, Struct, StructQuality, StructTag) +from matter_idl.matter_idl_types import (AccessPrivilege, ApiMaturity, Attribute, AttributeQuality, AttributeStorage, ClusterSide, + Command, CommandQuality, Event, EventPriority, EventQuality, FieldQuality, Idl, + StructQuality, StructTag) def human_text_string(value: Union[ClusterSide, StructTag, StructQuality, EventPriority, EventQuality, AccessPrivilege, AttributeQuality, CommandQuality, ApiMaturity, AttributeStorage]) -> str: diff --git a/scripts/py_matter_idl/matter_idl/test_idl_generator.py b/scripts/py_matter_idl/matter_idl/test_idl_generator.py index cc8e26a56bac62..ea1347fb0e7d4c 100755 --- a/scripts/py_matter_idl/matter_idl/test_idl_generator.py +++ b/scripts/py_matter_idl/matter_idl/test_idl_generator.py @@ -17,10 +17,7 @@ import os import sys import unittest -from dataclasses import dataclass, field -from typing import List - -import yaml +from typing import Optional try: from matter_idl.matter_idl_parser import CreateParser diff --git a/scripts/py_matter_idl/matter_idl/xml_parser.py b/scripts/py_matter_idl/matter_idl/xml_parser.py index 6fcd987d78661f..d20bdb9dda67ec 100755 --- a/scripts/py_matter_idl/matter_idl/xml_parser.py +++ b/scripts/py_matter_idl/matter_idl/xml_parser.py @@ -16,6 +16,7 @@ import logging import os +from typing import Optional try: from matter_idl.zapxml import ParseSource, ParseXmls