Add "provisional" and "internal" marker support for matter IDL files (#…

…29605)

* Add maturity markers to clusters and add unit tests

* Add maturity to cluster elements. For now tests for enum only

* Add tests for bitmap maturity

* Add unit tests for bitmap, event and struct api maturity

* Add more tests and support maturity for fields

* Add unit tests for event field maturity info

* Add api maturity for constants and unit tests for bitmaps in particular

* Add unit tests for maturity of enum constants

* Update readme about supported syntax

* Restyle

* Add deprecated maturity support

* Undo submodule update

* Add stable as a supported keyword and update a unit tst

---------

Co-authored-by: Andrei Litvin <[email protected]>

scripts/py_matter_idl/matter_idl/README.md

@@ -130,6 +130,23 @@ server cluster AccessControl = 31 {
 
   // commands may have multiple attributes
   fabric timed command RequiresTimedInvoke(): DefaultSuccess = 7;
+
+  // Items may have a prefix about api stability.
+  //   - "provisional" are generally subject to change
+  //   - "internal" are for internal SDK development/usage/testing
+
+  provisional critical event StartUp = 0 {
+       INT32U softwareVersion = 0;
+  }
+  internal struct SomeInternalStruct {}
+
+  struct StructThatIsBeingChanged {
+    CHAR_STRING debugText = 1;
+    provisional INT32S errorValue = 2;
+  }
+
+  provisional timedwrite attribute int16u attributeInDevelopment = 10;
+  internal command FactoryReset(): DefaultSuccess = 10;
 }
 
 // A client cluster represents something that is used by an app
@@ -150,6 +167,10 @@ client cluster OtaSoftwareUpdateProvider = 41 {
     ///.... content removed: it is very similar to a server cluster
 }
 
+// Clusters may be provisional or internal as well
+provisional client cluster SomeClusterInDevelopment = 1234 {
+  /// ... content removed
+}
 
 // On every endpoint number (non-dynamic)
 // a series of clusters can be exposed

scripts/py_matter_idl/matter_idl/matter_grammar.lark

@@ -2,6 +2,14 @@ struct: struct_qualities "struct"i id "{" (struct_field ";")* "}"
 struct_quality: "fabric_scoped"i -> struct_fabric_scoped
 struct_qualities: struct_quality*
 
+// Composing elements may be at different maturity level
+// This only contains non-stable items. Items without a maturity
+// flag are to be assumed "stable"
+?maturity: "provisional"i -> provisional_api_maturity
+         | "internal"i -> internal_api_maturity
+         | "deprecated"i -> deprecated_api_maturity
+         | "stable"i -> stable_api_maturity
+
 enum: "enum"i id ":" type "{" constant_entry* "}"
 bitmap: "bitmap"i id ":" type "{" constant_entry* "}"
 
@@ -53,9 +61,10 @@ command_with_access: "command"i command_access? id
 
 command: command_qualities command_with_access "(" id? ")" ":" id "=" positive_integer ";"
 
-cluster: cluster_side "cluster"i id "=" positive_integer "{" (enum|bitmap|event|attribute|struct|request_struct|response_struct|command)* "}"
+cluster: [maturity] cluster_side "cluster"i id "=" positive_integer "{" cluster_content* "}"
 ?cluster_side: "server"i -> server_cluster
              | "client"i -> client_cluster
+?cluster_content: [maturity] (enum|bitmap|event|attribute|struct|request_struct|response_struct|command)
 
 endpoint: "endpoint"i positive_integer "{" endpoint_content* "}"
 ?endpoint_content: endpoint_cluster_binding | endpoint_server_cluster | endpoint_device_type
@@ -78,13 +87,13 @@ bool_default: "true"i  -> bool_default_true
             | "false"i -> bool_default_false
 ?default_value: "default"i "=" (integer | ESCAPED_STRING | bool_default)
 
-constant_entry: id "=" positive_integer ";"
+constant_entry: [maturity] id "=" positive_integer ";"
 positive_integer: POSITIVE_INTEGER | HEX_INTEGER
 negative_integer: "-" positive_integer
 
 integer: positive_integer | negative_integer
 
-struct_field: member_attribute* field
+struct_field: [maturity] member_attribute* field
 
 member_attribute: "optional"i -> optional
                 | "nullable"i -> nullable

scripts/py_matter_idl/matter_idl/matter_idl_parser.py

@@ -18,10 +18,11 @@
 
     from matter_idl.matter_idl_types import AccessPrivilege
 
-from matter_idl.matter_idl_types import (Attribute, AttributeInstantiation, AttributeOperation, AttributeQuality, AttributeStorage,
-                                         Bitmap, Cluster, ClusterSide, Command, CommandQuality, ConstantEntry, DataType, DeviceType,
-                                         Endpoint, Enum, Event, EventPriority, EventQuality, Field, FieldQuality, Idl,
-                                         ParseMetaData, ServerClusterInstantiation, Struct, StructQuality, StructTag)
+from matter_idl.matter_idl_types import (ApiMaturity, Attribute, AttributeInstantiation, AttributeOperation, AttributeQuality,
+                                         AttributeStorage, Bitmap, Cluster, ClusterSide, Command, CommandQuality, ConstantEntry,
+                                         DataType, DeviceType, Endpoint, Enum, Event, EventPriority, EventQuality, Field,
+                                         FieldQuality, Idl, ParseMetaData, ServerClusterInstantiation, Struct, StructQuality,
+                                         StructTag)
 
 
 def UnionOfAllFlags(flags_list):
@@ -157,6 +158,18 @@ def bool_default_true(self, _):
     def bool_default_false(self, _):
         return False
 
+    def provisional_api_maturity(self, _):
+        return ApiMaturity.PROVISIONAL
+
+    def internal_api_maturity(self, _):
+        return ApiMaturity.INTERNAL
+
+    def deprecated_api_maturity(self, _):
+        return ApiMaturity.DEPRECATED
+
+    def stable_api_maturity(self, _):
+        return ApiMaturity.STABLE
+
     def id(self, tokens):
         """An id is a string containing an identifier
         """
@@ -181,8 +194,10 @@ def data_type(self, tokens):
             raise Exception("Unexpected size for data type")
 
     @v_args(inline=True)
-    def constant_entry(self, id, number):
-        return ConstantEntry(name=id, code=number)
+    def constant_entry(self, api_maturity, id, number):
+        if api_maturity is None:
+            api_maturity = ApiMaturity.STABLE
+        return ConstantEntry(name=id, code=number, api_maturity=api_maturity)
 
     @v_args(inline=True)
     def enum(self, id, type, *entries):
@@ -254,7 +269,9 @@ def struct_field(self, args):
         # Last argument is the named_member, the rest
         # are qualities
         field = args[-1]
-        field.qualities = UnionOfAllFlags(args[:-1]) or FieldQuality.NONE
+        field.qualities = UnionOfAllFlags(args[1:-1]) or FieldQuality.NONE
+        if args[0] is not None:
+            field.api_maturity = args[0]
         return field
 
     @v_args(meta=True)
@@ -445,15 +462,25 @@ def endpoint_server_cluster(self, meta, id, *content):
         return AddServerClusterToEndpointTransform(
             ServerClusterInstantiation(parse_meta=meta, name=id, attributes=attributes, events_emitted=events))
 
+    @v_args(inline=True)
+    def cluster_content(self, api_maturity, element):
+        if api_maturity is not None:
+            element.api_maturity = api_maturity
+        return element
+
     @v_args(inline=True, meta=True)
-    def cluster(self, meta, side, name, code, *content):
+    def cluster(self, meta, api_maturity, side, name, code, *content):
         meta = None if self.skip_meta else ParseMetaData(meta)
 
         # shift actual starting position where the doc comment would start
         if meta and self._cluster_start_pos:
             meta.start_pos = self._cluster_start_pos
 
-        result = Cluster(parse_meta=meta, side=side, name=name, code=code)
+        if api_maturity is None:
+            api_maturity = ApiMaturity.STABLE
+
+        result = Cluster(parse_meta=meta, side=side, name=name,
+                         code=code, api_maturity=api_maturity)
 
         for item in content:
             if type(item) == Enum:

scripts/py_matter_idl/matter_idl/matter_idl_types.py

@@ -5,8 +5,16 @@
 from lark.tree import Meta
 
 
+class ApiMaturity(enum.Enum):
+    STABLE = enum.auto()  # default
+    PROVISIONAL = enum.auto()
+    INTERNAL = enum.auto()
+    DEPRECATED = enum.auto()
+
 # Information about parsing location for specific items
 # Helpful when referencing data items in logs when processing
+
+
 @dataclass
 class ParseMetaData:
     line: Optional[int]
@@ -114,6 +122,7 @@ class Field:
     name: str
     is_list: bool = False
     qualities: FieldQuality = FieldQuality.NONE
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
     @property
     def is_optional(self):
@@ -131,6 +140,7 @@ class Attribute:
     readacl: AccessPrivilege = AccessPrivilege.VIEW
     writeacl: AccessPrivilege = AccessPrivilege.OPERATE
     default: Optional[Union[str, int]] = None
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
     @property
     def is_readable(self):
@@ -156,6 +166,7 @@ class Struct:
     tag: Optional[StructTag] = None
     code: Optional[int] = None  # for responses only
     qualities: StructQuality = StructQuality.NONE
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
 
 @dataclass
@@ -167,6 +178,7 @@ class Event:
     readacl: AccessPrivilege = AccessPrivilege.VIEW
     qualities: EventQuality = EventQuality.NONE
     description: Optional[str] = None
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
     @property
     def is_fabric_sensitive(self):
@@ -177,20 +189,23 @@ def is_fabric_sensitive(self):
 class ConstantEntry:
     name: str
     code: int
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
 
 @dataclass
 class Enum:
     name: str
     base_type: str
     entries: List[ConstantEntry]
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
 
 @dataclass
 class Bitmap:
     name: str
     base_type: str
     entries: List[ConstantEntry]
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
 
 @dataclass
@@ -202,6 +217,7 @@ class Command:
     qualities: CommandQuality = CommandQuality.NONE
     invokeacl: AccessPrivilege = AccessPrivilege.OPERATE
     description: Optional[str] = None
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
     # Parsing meta data missing only when skip meta data is requested
     parse_meta: Optional[ParseMetaData] = field(default=None)
@@ -223,6 +239,7 @@ class Cluster:
     structs: List[Struct] = field(default_factory=list)
     commands: List[Command] = field(default_factory=list)
     description: Optional[str] = None
+    api_maturity: ApiMaturity = ApiMaturity.STABLE
 
     # Parsing meta data missing only when skip meta data is requested
     parse_meta: Optional[ParseMetaData] = field(default=None)