Use ZAP to generate a more minimal code/reviewable IDL of what is ena…

…bled. (project-chip#13544)

* Start building a 'matter IDL' generator.

I would like to see what data is being generated WITHOUT trying
to parse code. This would make it easier on code reviews
and generally understanding what xml + zap actually mean.

* Some initial example: can generate clusters and attributes and a lot of  data types

* make fabric scoped by show up

* Add listing of commands

* make commands take arguments

* Only add cluster enums to the IDL

* Ran zap-all generate to get a lot of matter IDL files

* Support full command request/response and start describing endpoints

* Much better  display: only used structs and group by cluster and global as well

* regen all and this time the content seems smaller and focused

* Update FIXME to actual doc ... fixme was  done

* ZAP regen all again

* Restyle fixes

* Split longer line ifs onto separate lines

* Code review noticed readonly and readwrite were switched. Fix that

* Add support for flagging list attributes with []

* Add detection of array types, optionallity and nullability in struct items

* Optionality, nullability and list in request/response structs

* regen all

* Propper support for events, fix namings for structs

* Add indices to request and response parameters. Do not  include empty requests (they serve no purpose)

* re-ran zap, made some  things non-reportable

src/app/zap-templates/app-templates.json

@@ -26,6 +26,14 @@
         {
             "name": "im_command_handler_cluster_commands",
             "path": "partials/im_command_handler_cluster_commands.zapt"
+        },
+        {
+            "name": "idl_structure_definition",
+            "path": "partials/idl/structure_definition.zapt"
+        },
+        {
+            "name": "idl_structure_member",
+            "path": "partials/idl/structure_member.zapt"
         }
     ],
     "templates": [
@@ -78,6 +86,11 @@
             "path": "templates/app/CHIPClusters-src.zapt",
             "name": "C++ ZCL API",
             "output": "CHIPClusters.cpp"
+        },
+        {
+            "path": "templates/app/MatterIDL.zapt",
+            "name": "Human-readable Matter IDL",
+            "output": "Clusters.matter"
         }
     ]
 }

src/app/zap-templates/common/ClustersHelper.js

@@ -477,9 +477,105 @@ function enhancedAttributes(attributes, globalAttributes, types)
 }
 
 const Clusters = {
-  ready : new Deferred()
+  ready : new Deferred(),
+  post_processing_ready : new Deferred()
 };
 
+class ClusterStructUsage {
+  constructor()
+  {
+    this.usedStructures       = new Map(); // Structure label -> structure
+    this.clustersForStructure = new Map(); // Structure label -> Set(Cluster name)
+    this.structuresForCluster = new Map(); // Cluster name -> Set(Structure label)
+  }
+
+  addUsedStructure(clusterName, structure)
+  {
+    // Record that generally this structure is used
+    this.usedStructures.set(structure.label, structure);
+
+    // Record that this structure is used by a
+    // particular cluster name
+    let clusterSet = this.clustersForStructure.get(structure.label);
+    if (!clusterSet) {
+      clusterSet = new Set();
+      this.clustersForStructure.set(structure.label, clusterSet);
+    }
+    clusterSet.add(clusterName);
+
+    let structureLabelSet = this.structuresForCluster.get(clusterName);
+    if (!structureLabelSet) {
+      structureLabelSet = new Set();
+      this.structuresForCluster.set(clusterName, structureLabelSet);
+    }
+    structureLabelSet.add(structure.label);
+  }
+
+  /**
+   * Finds structures that are specific to one cluster:
+   *   - they belong to the cluster
+   *   - only that cluster ever uses it
+   */
+  structuresSpecificToCluster(clusterName)
+  {
+    let clusterStructures = this.structuresForCluster.get(clusterName);
+    if (!clusterStructures) {
+      return [];
+    }
+
+    return Array.from(clusterStructures)
+        .filter(name => this.clustersForStructure.get(name).size == 1)
+        .map(name => this.usedStructures.get(name));
+  }
+
+  structuresUsedByMultipleClusters()
+  {
+    return Array.from(this.usedStructures.values()).filter(s => this.clustersForStructure.get(s.label).size > 1);
+  }
+}
+
+Clusters._addUsedStructureNames = async function(clusterName, startType, allKnownStructs) {
+  const struct = getStruct(allKnownStructs, startType.type);
+  if (!struct) {
+    return;
+  }
+
+  this._cluster_structures.addUsedStructure(clusterName, struct);
+
+  for (const item of struct.items) {
+    this._addUsedStructureNames(clusterName, item, allKnownStructs);
+  }
+}
+
+Clusters._computeUsedStructureNames = async function(structs) {
+  // NOTE: this MUST be called only after attribute promise is resolved
+  // as iteration of `get*ByClusterName` needs that data.
+  for (const cluster of this._clusters) {
+    const attributes = await this.getAttributesByClusterName(cluster.name);
+    for (const attribute of attributes) {
+      if (attribute.isStruct) {
+        this._addUsedStructureNames(cluster.name, attribute, structs);
+      }
+    }
+
+    const commands = await this.getCommandsByClusterName(cluster.name);
+    for (const command of commands) {
+      for (const argument of command.arguments) {
+        this._addUsedStructureNames(cluster.name, argument, structs);
+      }
+    }
+
+    const responses = await this.getResponsesByClusterName(cluster.name);
+    for (const response of responses) {
+      for (const argument of response.arguments) {
+        this._addUsedStructureNames(cluster.name, argument, structs);
+      }
+    }
+  }
+
+  this._used_structure_names = new Set(this._cluster_structures.usedStructures.keys())
+}
+
 Clusters.init = async function(context) {
   if (this.ready.running)
   {
@@ -505,14 +601,20 @@ Clusters.init = async function(context) {
     loadEvents.call(context, packageId),
   ];
 
-  return Promise.all(promises).then(([types, clusters, commands, attributes, globalAttributes, events]) => {
-    this._clusters = clusters;
-    this._commands = enhancedCommands(commands, types);
-    this._attributes = enhancedAttributes(attributes, globalAttributes, types);
-    this._events = enhancedEvents(events, types);
+  let [types, clusters, commands, attributes, globalAttributes, events] = await Promise.all(promises);
+
+  this._clusters = clusters;
+  this._commands = enhancedCommands(commands, types);
+  this._attributes = enhancedAttributes(attributes, globalAttributes, types);
+  this._events = enhancedEvents(events, types);
+  this._cluster_structures = new ClusterStructUsage();
+
+  // data is ready, but not full post - processing
+  this.ready.resolve();
+
+  await this._computeUsedStructureNames(types[3]);
 
-    return this.ready.resolve();
-  }, err => this.ready.reject(err));
+  return this.post_processing_ready.resolve();
 }
 
 
@@ -545,6 +647,12 @@ Clusters.ensureReady = function()
     return this.ready;
 }
 
+Clusters.ensurePostProcessingDone = function()
+{
+    ensureState(this.ready.running);
+    return this.post_processing_ready;
+}
+
 Clusters.getClusters = function()
 {
     return this.ensureReady().then(() => this._clusters);
@@ -681,6 +789,21 @@ Clusters.getServerAttributes = function(name)
     return this.getAttributesByClusterName(name).then(items => items.filter(kServerSideFilter));
 }
 
+Clusters.getUsedStructureNames = function()
+{
+    return this.ensurePostProcessingDone().then(() => this._used_structure_names);
+}
+
+Clusters.getStructuresByClusterName = function(name)
+{
+    return this.ensurePostProcessingDone().then(() => this._cluster_structures.structuresSpecificToCluster(name));
+}
+
+Clusters.getSharedStructs = function()
+{
+    return this.ensurePostProcessingDone().then(() => this._cluster_structures.structuresUsedByMultipleClusters());
+}
+
 Clusters.getServerEvents = function(name)
 {
     return this.getEventsByClusterName(name).then(items => items.filter(kServerSideFilter));

src/app/zap-templates/partials/idl/structure_definition.zapt

@@ -0,0 +1,8 @@
+{{#if struct_is_fabric_scoped}}
+{{indent extraIndent~}} [fabric_scoped_by={{asUpperCamelCase struct_fabric_idx_field}}]
+{{/if}}
+{{indent extraIndent~}} struct {{name}} {
+{{#zcl_struct_items}}
+  {{indent extraIndent~}} {{> idl_structure_member}}
+{{/zcl_struct_items}}
+{{indent extraIndent~}} }

src/app/zap-templates/partials/idl/structure_member.zapt

@@ -0,0 +1,8 @@
+{{~#if isOptional~}} optional {{/if~}}
+{{~#if isNullable~}} nullable {{/if~}}
+
+{{type}} {{asLowerCamelCase label~}}
+
+{{~#if isArray~}}
+  []
+{{~/if}} = {{fieldIdentifier}};

src/app/zap-templates/templates/app/MatterIDL.zapt

@@ -0,0 +1,87 @@
+// This IDL was generated automatically by ZAP.
+// It is for view/code review purposes only.
+
+{{#chip_shared_structs}}
+{{>idl_structure_definition extraIndent=0}}
+
+{{/chip_shared_structs}}
+{{!TODO: consider #chip_clusters as iteration point as well.
+{{!      Unsure about the differences}}
+{{#all_user_clusters}}
+{{side}} cluster {{asUpperCamelCase name}} = {{code}} {
+  {{#zcl_enums}}
+  enum {{asUpperCamelCase name}} : {{type}} {
+    {{#zcl_enum_items}}
+    k{{asUpperCamelCase label}} = {{value}};
+    {{/zcl_enum_items}}
+  }
+
+  {{/zcl_enums}}
+  {{#chip_cluster_specific_structs}}
+{{>idl_structure_definition extraIndent=1}}
+
+  {{/chip_cluster_specific_structs}}
+  {{#zcl_events}}
+  {{priority}} event {{asUpperCamelCase name}} = {{code}} {
+    {{#zcl_event_fields}}
+    {{>idl_structure_member label=name}}
+
+    {{/zcl_event_fields}}
+  }
+
+  {{/zcl_events}}
+  {{#chip_cluster_attributes}}
+  attribute(
+    {{~#if isWritableAttribute~}}
+      writable
+    {{~else~}}
+      readonly
+    {{~/if~}}
+    {{~#if isReportableAttribute~}}
+      , reportable
+    {{~/if~}}
+    ) {{type}} {{asLowerCamelCase name~}}
+    {{~#if isList~}}
+      []
+    {{~/if}} = {{code}};
+  {{/chip_cluster_attributes}}
+  {{#chip_cluster_commands}}
+  {{#if arguments}}
+
+  request struct {{asUpperCamelCase name}}Request {
+    {{#chip_cluster_command_arguments}}
+    {{> idl_structure_member}}
+
+    {{/chip_cluster_command_arguments}}
+  }
+  {{/if}}
+  {{/chip_cluster_commands}}
+  {{#chip_cluster_responses}}
+
+  response struct {{asUpperCamelCase name}} {
+    {{#chip_cluster_response_arguments}}
+    {{> idl_structure_member}}
+
+    {{/chip_cluster_response_arguments}}
+  }
+  {{/chip_cluster_responses}}
+  {{#chip_cluster_commands}}
+  {{#first}}
+
+  {{/first}}
+  command {{asUpperCamelCase name}}(
+    {{~#if arguments~}}
+       {{asUpperCamelCase name}}Request
+    {{~/if~}}
+  ): {{asUpperCamelCase responseName}} = {{code}};
+  {{/chip_cluster_commands}}
+}
+
+{{/all_user_clusters}}
+
+{{#user_endpoints}}
+endpoint {{endpointId}} {
+    {{!TODO: report device types and cluster instantionation. }}
+}
+
+{{/user_endpoints}}

src/app/zap-templates/templates/chip/helper.js

@@ -84,6 +84,12 @@ function getCommands(methodName)
                                  : ensureClusters(this).getServerCommands(clusterName);
 }
 
+function getAttributes(methodName)
+{
+  const { clusterName, clusterSide } = checkIsInsideClusterBlock(this, methodName);
+  return ensureClusters(this).getAttributesByClusterName(clusterName);
+}
+
 function getResponses(methodName)
 {
   const { clusterName, clusterSide } = checkIsInsideClusterBlock(this, methodName);
@@ -215,6 +221,20 @@ function chip_cluster_commands(options)
   return asBlocks.call(this, commands, options);
 }
 
+/**
+ * Creates block iterator over the cluster attributes for a given cluster/side.
+ *
+ * This function is meant to be used inside a {{#chip_*_clusters}}
+ * block. It will throw otherwise.
+ *
+ * @param {*} options
+ */
+function chip_cluster_attributes(options)
+{
+  const attributes = getAttributes.call(this, 'chip_cluster_attributes');
+
+  return asBlocks.call(this, attributes, options);
+}
 /**
  * Creates block iterator over the cluster responses for a given cluster/side.
  *
@@ -244,7 +264,8 @@ function chip_cluster_command_arguments(options)
   const commands  = getCommands.call(this.parent, 'chip_cluster_commands_argments');
 
   const filter = command => command.id == commandId;
-  return asBlocks.call(this, commands.then(items => items.find(filter).arguments), options);
+  return asBlocks.call(this,
+      commands.then(items => items.find(filter).arguments.map((value, index) => ({...value, fieldIdentifier : index }))), options);
 }
 
 /**
@@ -293,7 +314,8 @@ function chip_cluster_response_arguments(options)
   const responses = getResponses.call(this.parent, 'chip_cluster_responses_argments');
 
   const filter = command => command.id == commandId;
-  return asBlocks.call(this, responses.then(items => items.find(filter).arguments), options);
+  return asBlocks.call(this,
+      responses.then(items => items.find(filter).arguments.map((value, index) => ({...value, fieldIdentifier : index }))), options);
 }
 
 /**
@@ -391,6 +413,27 @@ function chip_available_cluster_commands(options)
   return promise;
 }
 
+/**
+ * Creates block iterator over structures belonging to the current cluster
+ */
+async function chip_cluster_specific_structs(options)
+{
+  const { clusterName, clusterSide } = checkIsInsideClusterBlock(this, 'chip_cluster_specific_structs');
+
+  const structs = await ensureClusters(this).getStructuresByClusterName(clusterName);
+
+  return templateUtil.collectBlocks(structs, options, this);
+}
+
+/**
+ * Creates block iterator over structures that are shared between clusters
+ */
+async function chip_shared_structs(options)
+{
+  const structs = await ensureClusters(this).getSharedStructs();
+  return templateUtil.collectBlocks(structs, options, this);
+}
+
 /**
  * Checks whether a type is an enum for purposes of its chipType.  That includes
  * both spec-defined enum types and types that we map to enum types in our code.
@@ -424,6 +467,7 @@ exports.chip_has_server_clusters                             = chip_has_server_c
 exports.chip_cluster_commands                                = chip_cluster_commands;
 exports.chip_cluster_command_arguments                       = chip_cluster_command_arguments;
 exports.chip_cluster_command_arguments_with_structs_expanded = chip_cluster_command_arguments_with_structs_expanded;
+exports.chip_cluster_attributes                              = chip_cluster_attributes;
 exports.chip_server_global_responses                         = chip_server_global_responses;
 exports.chip_cluster_responses                               = chip_cluster_responses;
 exports.chip_cluster_response_arguments                      = chip_cluster_response_arguments
@@ -434,3 +478,5 @@ exports.chip_server_has_list_attributes                      = chip_server_has_l
 exports.chip_available_cluster_commands                      = chip_available_cluster_commands;
 exports.if_chip_enum                                         = if_chip_enum;
 exports.if_in_global_responses                               = if_in_global_responses;
+exports.chip_cluster_specific_structs                        = chip_cluster_specific_structs;
+exports.chip_shared_structs                                  = chip_shared_structs;