Allow IDLs to define storage info (callbacks and default value) (#17787)

* Add a README.md entry for desirable storage access info

* Start adding unit tests (they will fail)

* Grammar update: put semicolor outside of struct field so we can inject attributes

* Grammar update: support traits

* Unit tests pass

* Restyle

* Add accessor method for isCallback

* Pretty print by default in the idl parser - much more viewable data

* Some more keywords can be case insensitive just in case

* remove strange comment tag - I do not believe this was ever used

* Add `persist` support

* Fix typo

* Restyle

* Update scripts/idl/README.md

Co-authored-by: Tennessee Carmel-Veilleux <[email protected]>

Co-authored-by: Tennessee Carmel-Veilleux <[email protected]>

scripts/idl/README.md

@@ -97,6 +97,24 @@ server cluster AccessControl = 31 {
   // attributes may be read-only as well
   readonly attribute int16u clusterRevision = 65533;
 
+  // Code-gen specific information for storage can be added as a keyword list
+  // after the attribute number. Specifically these are supported
+  //   - `callback` is the equivalent of EXTERNAL in ember/zap, which means
+  //     the value does not have a RAM backing store (use callbacks for get/set)
+  //   - `persist` is the equivalent of NVM in ember/zap, which means
+  //     the value will be persisted to storage when written (boot-time restores
+  //     any set value)
+  //   - `default` is supported to set a default value in RAM-based attribute store
+  //
+  // Not all combination of values are compatible. In particular:
+  //  - `callback` is incompatible with `default` or `persist` as all value
+  //    computation is deferred to the app.
+  readonly attribute int16u usingExternalAccess = 10 [callback];
+  readonly attribute int16u isPersisted = 10 [persist];
+  readonly attribute int16u hasDefaultValue = 11 [default=123];
+  readonly attribute char_string<16> defaultStringValue = 12 [default="abc"];
+
+
   // Commands have spec-defined numbers which are used for over-the-wire
   // invocation.
   //

scripts/idl/matter_grammar.lark

@@ -1,4 +1,4 @@
-struct: "struct"i id "{" struct_field* "}"
+struct: "struct"i id "{" (struct_field ";")* "}"
 enum: "enum"i id ":" type "{" constant_entry* "}"
 bitmap: "bitmap"i id ":" type "{" constant_entry* "}"
 
@@ -14,34 +14,42 @@ event_access: "access" "(" ("read" ":" access_privilege)? ")"
 
 event_with_access: "event" event_access? id
 
-event: event_priority event_with_access "=" number "{" struct_field* "}"
+event: event_priority event_with_access "=" number "{" (struct_field ";")* "}"
 
 ?event_priority: "critical"i -> critical_priority
                | "info"i     -> info_priority
                | "debug"i    -> debug_priority
 
 attribute_access_entry: attribute_access_type ":" access_privilege
 
-attribute_access: "access" "(" (attribute_access_entry ("," attribute_access_entry)* )? ")"
+default_value: "default"i "=" (number | ESCAPED_STRING)
+
+attribute_is_callback: "callback"i
+attribute_is_persist: "persist"i
+?single_attribute_trait: attribute_is_callback | attribute_is_persist | default_value
+
+attribute_traits: "[" single_attribute_trait ("," single_attribute_trait)* "]"
+
+attribute_access: "access"i "(" (attribute_access_entry ("," attribute_access_entry)* )? ")"
 
 attribute_with_access: attribute_access? struct_field
 
-attribute: attribute_tag* "attribute"i attribute_with_access
+attribute: attribute_tag* "attribute"i attribute_with_access attribute_traits? ";"
 attribute_tag: "readonly"i -> attr_readonly
              | "nosubscribe"i -> attr_nosubscribe
 
 request_struct: "request"i struct
 
 // Response structures must have a response id
-response_struct: "response"i "struct"i id "=" number "{" struct_field * "}"
+response_struct: "response"i "struct"i id "=" number "{" (struct_field ";") * "}"
 
 command_attribute: "timed"i -> timed_command
 command_attributes: command_attribute*
 
 
-command_access: "access" "(" ("invoke" ":" access_privilege)? ")"
+command_access: "access"i "(" ("invoke"i ":" access_privilege)? ")"
 
-command_with_access: "command" command_access? id
+command_with_access: "command"i command_access? id
 
 command: command_attributes command_with_access "(" id? ")" ":" id "=" number ";"
 
@@ -63,23 +71,21 @@ struct_field: member_attribute* field
 member_attribute: "optional"i -> optional
                 | "nullable"i -> nullable
 
-field: data_type id list_marker? "=" number ";"
+field: data_type id list_marker? "=" number
 list_marker: "[" "]"
 
 data_type: type ("<" number ">")?
 
 id: ID
 type: ID
 
-COMMENT: "{" /(.|\n)+/ "}"
-       | "//" /.*/
-
 POSITIVE_INTEGER: /\d+/ 
 HEX_INTEGER: /0x[A-Fa-f0-9]+/ 
 ID: /[a-zA-Z_][a-zA-Z0-9_]*/
 
 idl: (struct|enum|cluster|endpoint)*
 
+%import common.ESCAPED_STRING
 %import common.WS
 %import common.C_COMMENT
 %import common.CPP_COMMENT

scripts/idl/matter_idl_parser.py

@@ -15,6 +15,30 @@
     from matter_idl_types import *
 
 
+class AttributeTransformDefaultValue:
+    def __init__(self, value):
+        self.value = value
+
+    def __call__(self, attr):
+        attr.default = self.value
+
+
+class AttributeTransformSetCallback:
+    def __init__(self):
+        pass
+
+    def __call__(self, attr):
+        attr.tags.add(AttributeTag.CALLBACK)
+
+
+class AttributeTransformSetPersisted:
+    def __init__(self):
+        pass
+
+    def __call__(self, attr):
+        attr.tags.add(AttributeTag.PERSIST)
+
+
 class MatterIdlTransformer(Transformer):
     """
     A transformer capable to transform data parsed by Lark according to 
@@ -227,12 +251,39 @@ def attribute_with_access(self, args):
 
         return (args[-1], acl)
 
+    def ESCAPED_STRING(self, s):
+        # handle escapes, skip the start and end quotes
+        return s.value[1:-1].encode('utf-8').decode('unicode-escape')
+
+    @v_args(inline=True)
+    def default_value(self, value):
+        return AttributeTransformDefaultValue(value)
+
+    @v_args(inline=True)
+    def attribute_is_callback(self):
+        return AttributeTransformSetCallback()
+
+    @v_args(inline=True)
+    def attribute_is_persist(self):
+        return AttributeTransformSetPersisted()
+
+    def attribute_traits(self, traits):
+        # traits as is as a list
+        return traits
+
     def attribute(self, args):
         # Input arguments are:
-        #   -  tags (0 or more)
-        #   -  attribute_with_access (i.e. pair of definition and acl arguments)
-        tags = set(args[:-1])
-        (definition, acl) = args[-1]
+        #   - tags (0 or more)
+        #   - attribute_with_access (i.e. pair of definition and acl arguments)
+        #   - attribute traits (last element)
+        if type(args[-1]) is tuple:
+            tags = set(args[:-1])
+            (definition, acl) = args[-1]
+            extra_attrs = []
+        else:
+            tags = set(args[:-2])
+            (definition, acl) = args[-2]
+            extra_attrs = args[-1]
 
         # until we support write only (and need a bit of a reshuffle)
         # if the 'attr_readonly == READABLE' is not in the list, we make things
@@ -241,7 +292,11 @@ def attribute(self, args):
             tags.add(AttributeTag.READABLE)
             tags.add(AttributeTag.WRITABLE)
 
-        return Attribute(definition=definition, tags=tags, **acl)
+        attr = Attribute(definition=definition, tags=tags, **acl)
+        for f in extra_attrs:
+            f(attr)
+
+        return attr
 
     @v_args(inline=True)
     def struct(self, id, *fields):
@@ -326,6 +381,7 @@ def CreateParser():
     # The ability to run is for debug and to print out the parsed AST.
     import click
     import coloredlogs
+    import pprint
 
     # Supported log levels, mapping string values required for argument
     # parsing into logging constants
@@ -352,6 +408,6 @@ def main(log_level, filename=None):
         logging.info("Parse completed")
 
         logging.info("Data:")
-        print(data)
+        pprint.pp(data)
 
     main()

scripts/idl/matter_idl_types.py

@@ -1,7 +1,7 @@
 import enum
 
 from dataclasses import dataclass, field
-from typing import List, Set, Optional
+from typing import List, Set, Optional, Union
 
 
 class FieldAttribute(enum.Enum):
@@ -17,6 +17,8 @@ class AttributeTag(enum.Enum):
     READABLE = enum.auto()
     WRITABLE = enum.auto()
     NOSUBSCRIBE = enum.auto()
+    CALLBACK = enum.auto()
+    PERSIST = enum.auto()
 
 
 class EventPriority(enum.Enum):
@@ -83,6 +85,7 @@ class Attribute:
     tags: Set[AttributeTag] = field(default_factory=set)
     readacl: AccessPrivilege = AccessPrivilege.VIEW
     writeacl: AccessPrivilege = AccessPrivilege.OPERATE
+    default: Optional[Union[str, int]] = None
 
     @property
     def is_readable(self):
@@ -96,6 +99,10 @@ def is_writable(self):
     def is_subscribable(self):
         return AttributeTag.NOSUBSCRIBE not in self.tags
 
+    @property
+    def is_callback(self):
+        return AttributeTag.CALLBACK in self.tags
+
 
 @dataclass
 class Struct:

scripts/idl/test_matter_idl_parser.py

@@ -180,6 +180,42 @@ def test_attribute_access(self):
                     )])
         self.assertEqual(actual, expected)
 
+    def test_attribute_storage_info(self):
+        actual = parseText("""
+            server cluster MyCluster = 1 {
+                attribute char_string<11> attr1 = 1 [callback];
+                attribute char_string<33> attr2 = 2 [default="abc\\n with escapes: \\""];
+                attribute int32u withDefault = 3 [default=11];
+                attribute int32u intWithCallback = 4 [callback];
+                attribute int32u persisted = 5 [persist];
+                attribute int32u persisted_with_default = 6 [persist, default=55];
+                readonly attribute int32u readonlyDefault = 7 [default=321];
+            }
+        """)
+
+        expected = Idl(clusters=[
+            Cluster(side=ClusterSide.SERVER,
+                    name="MyCluster",
+                    code=1,
+                    attributes=[
+                        Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.CALLBACK]), definition=Field(
+                            data_type=DataType(name="char_string", max_length=11), code=1, name="attr1")),
+                        Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field(
+                            data_type=DataType(name="char_string", max_length=33), code=2, name="attr2"), default='abc\n with escapes: "'),
+                        Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field(
+                            data_type=DataType(name="int32u"), code=3, name="withDefault"), default=11),
+                        Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.CALLBACK]), definition=Field(
+                            data_type=DataType(name="int32u"), code=4, name="intWithCallback")),
+                        Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.PERSIST]), definition=Field(
+                            data_type=DataType(name="int32u"), code=5, name="persisted")),
+                        Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.PERSIST]), definition=Field(
+                            data_type=DataType(name="int32u"), code=6, name="persisted_with_default"), default=55),
+                        Attribute(tags=set([AttributeTag.READABLE]), definition=Field(
+                            data_type=DataType(name="int32u"), code=7, name="readonlyDefault"), default=321),
+                    ]
+                    )])
+        self.assertEqual(actual, expected)
+
     def test_cluster_commands(self):
         actual = parseText("""
             server cluster WithCommands = 1 {