Mei data in all clusters (#19274)

* Initial commit to add some MEI types - specifically a MEI attribute

* Update code to be non-hex

* Add an extensions xml file

* Added a manufacturer specific extension, codegen seems to work on all clusters app

* ran zap regen after adding custom command in zap file

* Move extensions to separate json ... does not work in generation yet though

* Detect zcl path from zap file during generation

* Slight update with comment update

* Update code to generate MEI command

* Update comment

* Remove custom command - all clusters fails to compile with that. unsure why however assuming constants generated separately in different codegen paths

* Restyle

* Make run_zaptool.sh be able to use test extensions for all clusters app

* add zcl file detection for convert.py as well

* Add json import

* Ran convert.py on all-clusters

* Restyle

* Add extra slash to check for file path for all clusters

* Add linter rule for ZCL file being in sync

* Restyle

.github/workflows/lint.yml

@@ -85,6 +85,10 @@ jobs:
               if: always()
               run: scripts/tools/check_includes.sh
 
+            - name: Check for zcl.json and extension sync status
+              if: always()
+              run: scripts/tools/check_zcl_file_sync.py .
+
             - name: Ensure all PICS are set for tests (to true or false)
               if: always()
               run: |

examples/all-clusters-app/all-clusters-common/all-clusters-app.matter

@@ -2170,6 +2170,7 @@ server cluster ModeSelect = 80 {
   readonly attribute attrib_id attributeList[] = 65531;
   readonly attribute bitmap32 featureMap = 65532;
   readonly attribute int16u clusterRevision = 65533;
+  readonly attribute int8u manufacturerExtension = 4293984257;
 
   request struct ChangeToModeRequest {
     INT8U newMode = 0;
@@ -4328,6 +4329,7 @@ endpoint 1 {
     callback attribute attributeList;
     ram      attribute featureMap default = 1;
     ram      attribute clusterRevision default = 1;
+    ram      attribute manufacturerExtension default = 255;
   }
 
   server cluster DoorLock {

examples/all-clusters-app/all-clusters-common/all-clusters-app.zap

@@ -18,7 +18,7 @@
   "package": [
     {
       "pathRelativity": "relativeToZap",
-      "path": "../../../src/app/zap-templates/zcl/zcl.json",
+      "path": "../../../src/app/zap-templates/zcl/zcl-with-test-extensions.json",
       "version": "ZCL Test Data",
       "type": "zcl-properties"
     },
@@ -12664,6 +12664,22 @@
               "minInterval": 1,
               "maxInterval": 65534,
               "reportableChange": 0
+            },
+            {
+              "name": "ManufacturerExtension",
+              "code": 4293984257,
+              "mfgCode": null,
+              "side": "server",
+              "type": "int8u",
+              "included": 1,
+              "storageOption": "RAM",
+              "singleton": 0,
+              "bounded": 0,
+              "defaultValue": "255",
+              "reportable": 1,
+              "minInterval": 1,
+              "maxInterval": 65534,
+              "reportableChange": 0
             }
           ]
         },

scripts/tools/check_zcl_file_sync.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Validates that the json zcl files that are used by the app are in sync.
+
+Current rule:
+  - all-clusters app uses an extension on top of the "standard" zcl file.
+    Ensure that the two fies are in sync EXCEPT the extension.
+"""
+
+import json
+import sys
+import os
+import difflib
+
+
+def main():
+    if len(sys.argv) != 2:
+        print('Please pass CHIP_ROOT as an argument (and only chip root)')
+        return 1
+
+    base_name = os.path.join(sys.argv[1], "src", "app", "zap-templates", "zcl", "zcl.json")
+    ext_name = os.path.join(sys.argv[1], "src", "app", "zap-templates", "zcl", "zcl-with-test-extensions.json")
+
+    base_data = json.load(open(base_name))
+    ext_data = json.load(open(ext_name))
+
+    # ext should be IDENTICAL with base if we add a few things to base:
+    base_data["xmlRoot"].append("./data-model/test")
+    base_data["xmlFile"].append("mode-select-extensions.xml")
+
+    # do not care about sorting. mainly do not check if extension xml
+    # is at the end or in the middle
+    base_data["xmlFile"].sort()
+    ext_data["xmlFile"].sort()
+
+    if base_data == ext_data:
+        return 0
+
+    print("%s and %s have unexpected differences." % (base_name, ext_name))
+    print("Differences between expected and actual:")
+
+    for l in difflib.unified_diff(
+        json.dumps(ext_data, indent=2).split('\n'),
+        json.dumps(base_data, indent=2).split('\n'),
+        fromfile=ext_name,
+        tofile="<Expected extension file content>",
+    ):
+        if l.endswith('\n'):
+            l = l[:-1]
+        print(l)
+
+    return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/tools/zap/convert.py

@@ -16,6 +16,7 @@
 #
 
 import argparse
+import json
 import os
 import subprocess
 import sys
@@ -66,9 +67,28 @@ def runArgumentsParser():
     return zap_file
 
 
+def detectZclFile(zapFile):
+    print(f"Searching for zcl file from {zapFile}")
+
+    path = 'src/app/zap-templates/zcl/zcl.json'
+
+    data = json.load(open(zapFile))
+    for package in data["package"]:
+        if package["type"] != "zcl-properties":
+            continue
+
+        # found the right path, try to figure out the actual path
+        if package["pathRelativity"] == "relativeToZap":
+            path = os.path.abspath(os.path.join(os.path.dirname(zapFile), package["path"]))
+        else:
+            path = package["path"]
+
+    return getFilePath(path)
+
+
 def runConversion(zap_file):
     templates_file = getFilePath('src/app/zap-templates/app-templates.json')
-    zcl_file = getFilePath('src/app/zap-templates/zcl/zcl.json')
+    zcl_file = detectZclFile(zap_file)
 
     generator_dir = getDirPath('third_party/zap/repo')
     os.chdir(generator_dir)

scripts/tools/zap/generate.py

@@ -58,18 +58,36 @@ def getDirPath(name):
     return fullpath
 
 
+def detectZclFile(zapFile):
+    print(f"Searching for zcl file from {zapFile}")
+
+    path = 'src/app/zap-templates/zcl/zcl.json'
+
+    data = json.load(open(zapFile))
+    for package in data["package"]:
+        if package["type"] != "zcl-properties":
+            continue
+
+        # found the right path, try to figure out the actual path
+        if package["pathRelativity"] == "relativeToZap":
+            path = os.path.abspath(os.path.join(os.path.dirname(zapFile), package["path"]))
+        else:
+            path = package["path"]
+
+    return getFilePath(path)
+
+
 def runArgumentsParser():
     default_templates = 'src/app/zap-templates/app-templates.json'
-    default_zcl = 'src/app/zap-templates/zcl/zcl.json'
     default_output_dir = 'zap-generated/'
 
     parser = argparse.ArgumentParser(
         description='Generate artifacts from .zapt templates')
     parser.add_argument('zap', help='Path to the application .zap file')
     parser.add_argument('-t', '--templates', default=default_templates,
                         help='Path to the .zapt templates records to use for generating artifacts (default: "' + default_templates + '")')
-    parser.add_argument('-z', '--zcl', default=default_zcl,
-                        help='Path to the zcl templates records to use for generating artifacts (default: "' + default_zcl + '")')
+    parser.add_argument('-z', '--zcl',
+                        help='Path to the zcl templates records to use for generating artifacts (default: autodetect read from zap file)')
     parser.add_argument('-o', '--output-dir', default=None,
                         help='Output directory for the generated files (default: automatically selected)')
     args = parser.parse_args()
@@ -86,7 +104,12 @@ def runArgumentsParser():
         output_dir = ''
 
     zap_file = getFilePath(args.zap)
-    zcl_file = getFilePath(args.zcl)
+
+    if args.zcl:
+        zcl_file = getFilePath(args.zcl)
+    else:
+        zcl_file = detectZclFile(zap_file)
+
     templates_file = getFilePath(args.templates)
     output_dir = getDirPath(output_dir)
 

scripts/tools/zap/run_zaptool.sh

@@ -45,9 +45,16 @@ CHIP_ROOT="${SCRIPT_PATH%/scripts/tools/zap/run_zaptool.sh}"
         npm install
     fi
 
+    echo "ARGS: ${ZAP_ARGS[@]}"
+
+    if [[ "${ZAP_ARGS[@]}" == *"/all-clusters-app.zap"* ]]; then
+        ZCL_FILE="$CHIP_ROOT/src/app/zap-templates/zcl/zcl-with-test-extensions.json"
+    else
+        ZCL_FILE="$CHIP_ROOT/src/app/zap-templates/zcl/zcl.json"
+    fi
+
     node src-script/zap-start.js --logToStdout \
         --gen "$CHIP_ROOT/src/app/zap-templates/app-templates.json" \
-        --zcl "$CHIP_ROOT/src/app/zap-templates/zcl/zcl.json" \
+        --zcl "$ZCL_FILE" \
         "${ZAP_ARGS[@]}"
-
 )

src/app/zap-templates/zcl/data-model/test/mode-select-extensions.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!--
+Copyright (c) 2022 Project CHIP Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configurator>
+  <domain name="CHIP"/>
+  <!-- This extends the ModeSelect cluster -->
+  <clusterExtension code="0x0050">
+    <!--
+       Manufacturer specific extension attribute:
+          - Prefix: 0xFFF1 - Test vendor MC
+          - Suffix: 0x0001 - Non-global attribute id
+     -->
+    <attribute side="server" code="0xFFF10001" define="MEI_EXTENSION" type="INT8U" writable="false" optional="true" isNullable="false">ManufacturerExtension</attribute>
+
+    <!--
+       Manufacturer specific extension command:
+          - Prefix: 0xFFF1 - Test vendor MC
+          - Suffix: 0x0001 - command id
+     -->
+    <command source="client" code="0xFFF10001" name="SampleMfgExtensionCommand" optional="true">
+      <description>Sample manufacturer specific extension command</description>
+    </command>
+  </clusterExtension>
+</configurator>