Chef - Add sample_app_util for parsing zap files (#19087)

.restyled.yaml

@@ -70,6 +70,7 @@ exclude:
     - "src/controller/python/chip/clusters/Objects.py" # generated file, no point to restyle
     - "src/controller/python/chip/clusters/CHIPClusters.py" # generated file, no point to restyle
     - "scripts/idl/tests/outputs/**/*" # Matches generated output 1:1
+    - "examples/chef/sample_app_util/test_files/*.yaml"
     - "examples/chef/zzz_generated/**/*"
 
 

examples/chef/BUILD.gn

@@ -21,12 +21,24 @@ import("$dir_pw_build/python.gni")
 pw_python_package("chef") {
   setup = [ "setup.py" ]
 
+  inputs = [
+    "sample_app_util/test_files/sample_zap_file.zap",
+    "sample_app_util/test_files/sample_zap_file_hashmeta.yaml",
+  ]
+
   sources = [
     "__init__.py",
     "chef.py",
     "constants.py",
+    "sample_app_util/__init__.py",
+    "sample_app_util/sample_app_util.py",
+    "sample_app_util/test_zap_file_parser.py",
+    "sample_app_util/zap_file_parser.py",
     "stateful_shell.py",
   ]
 
-  tests = [ "test_stateful_shell.py" ]
+  tests = [
+    "test_stateful_shell.py",
+    "sample_app_util/test_zap_file_parser.py",
+  ]
 }

examples/chef/sample_app_util/README.md

@@ -0,0 +1,195 @@
+# Chef Build Conventions
+
+## Overview
+
+---
+
+It is convenient to follow some naming and build conventions for Chef tool due
+to the large volume of sample apps that may be created and the ambiguity that
+may result from arbitrary names.
+
+There are three components to the convention proposed here:
+
+1. The naming convention for the sample matter device and clusters (referred to
+   here as the `sample app`).
+2. The naming convention to use for the build files which will be flashed on the
+   devices.
+3. The usage of metadata files that shall accompany build files to provide more
+   detailed information about builds.
+
+The convention proposed here should be adopted by the zap files provided in
+`examples/chef/devices` and the builds generated from Chef tool in CI.
+
+## Limitations
+
+---
+
+The largest filename that can be used on MacOS and Linux is 255 characters. If a
+sample app name would exceed this limit by following this convention, then the
+sample app should be given an arbitrary name.
+
+This limitation is called out, but, with the given naming conventions, this
+should rarely happen.
+
+## Convention
+
+---
+
+### Sample App Naming Convention
+
+Sample apps should be named by concatenating the name of all endpoints in the
+order of their index. Endpoint names are separated by underscores (`_`) and a 10
+character hash[^hash_note] of the sample app metadata is appended to the end.
+
+Valid sample app names conform to the following format:
+
+```
+<endpoint_0>_<endpoint_1>_<hash>
+```
+
+For example, here are some valid names:
+
+```
+rootnode_extendedcolorlight_H1l9gnQDYl
+rootnode_speaker_8qRQaEj0Hy
+rootnode_lightsensor_L6dEbmVDah
+rootnode_dimmablelight_rWsDiwzw2t
+rootnode_pressuresensor_03quf7tPOL
+rootnode_flowsensor_ixbAboycie
+rootnode_windowcovering_b9QoiScjOq
+rootnode_doorlock_d5wtU7sjFR
+rootnode_thermostat_KuQYArmwl7
+rootnode_dimmablelight_7pNE3GVarn
+rootnode_temperaturesensor_i0wGnDVUAc
+rootnode_occupancysensor_wyGeQSokNp
+rootnode_humiditysensor_pv0comNKyT
+bridgednode_temperaturesensor_onofflight_onoffpluginunit_MI9DSdkH8H
+```
+
+[^hash_note]:
+
+    The 10 character hash is a base64 encoding of the md5 hash generated by
+    digesting the JSON string encoding of the metadata information. The code for
+    generating the hash can be found in `generate_hash` in
+    [zap_file_parser](zap_file_parser.py) There are some notable details
+    here: 1) The full base64 encoded hash is 16 characters, but only 10 are
+    used. This still gives us a sufficiently low probability of collision (~1.2
+    x 10^-8). 2) `_` and `-` are replaced in the base64 encoding because they
+    have other uses in the naming. 3) Platform specific information is omitted
+    from the hash. E.g. the networking_commissioning cluster is excluded. This
+    is to make the hashes platform agnostic.
+
+### Sample App Build Naming Convention
+
+The sample app builds formats will be named by pre-pending the zap file name
+(described above) with the platform and appending connectivity info.
+
+Valid build names conform to the following format:
+
+```
+<platform>_<sample_app_name>
+```
+
+Note that `<sample_app_name>` follows the convention:
+`<endpoint_0>_<endpoint_1>_<hash>`.
+
+Together that is:
+
+```
+<platform>_<endpoint_0>_<endpoint_1>_<hash>
+```
+
+The list of platforms supported here (as of writing this) are:
+
+```
+m5stack
+brd4161a
+nrf52840dk
+linux_x86
+```
+
+For example, here are some valid names:
+
+```
+m5stack_rootnode_humiditysensor_pv0comNKyT
+brd4161a_rootnode_humiditysensor_pv0comNKyT
+nrf52840dk_rootnode_humiditysensor_pv0comNKyT
+linux_x86_rootnode_humiditysensor_pv0comNKyT
+```
+
+### Metadata file convention
+
+Metadata files are `yaml` files that should accompany build files.
+
+The metadata files have a structure as follows:
+
+```
+- <endpoint_0_name>:
+      client_clusters:
+          <client_cluster_name>:
+              attributes:
+                  <attribute_name>: <attribute_value>
+                  ...
+              commands:
+                  - <command_name>
+                  - ...
+      server_clusters:
+          <server_cluster_name>:
+              attributes:
+                  <attribute_name>: <attribute_value>
+                  ...
+              commands:
+                  - <command_name>
+                  - ...
+- <endpoint_1_name>: ...
+```
+
+For an example, see [sample_zap_file.yaml](test_files/sample_zap_file.yaml)
+which was generated from [sample_zap_file.zap](test_files/sample_zap_file.zap).
+
+Note that it is more readable in `yaml` format. Since hashes are generated from
+the metadata info, additional conventions are needed to ensure consistency for
+the metadata structure.
+
+The following conventions are used:
+
+-   All lists are sorted alphabetically.
+-   If a list contains dictionaries, it will be sorted by the "name" key. If it
+    does not contain "name" key, it will be sorted by the first key common to
+    all dictionaries that comes first alphabetically.
+-   The list of endpoints is excluded from the above conventions. Endpoints are
+    ordered according to their endpoint number; here, the endpoint number is the
+    same as the order they are read from the zap file.
+
+As an example, take a look at
+[sample_zap_file.yaml](test_files/sample_zap_file.yaml)
+
+## Utility Usage
+
+---
+
+There are a few primary usage cases for the utility
+[sample_app_util.py](sample_app_util.py). Details are provided by using
+`python sample_app_util.py zap --help`. Below is a summary.
+
+| Command                                                                      | Description                                                                                                                                                                  |
+| ---------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `python sample_app_util.py zap <zap_file> --generate-name`                   | Generates the name for a zap file per the specified convention                                                                                                               |
+| `python sample_app_util.py zap <zap_file> --rename-file`                     | Renames the zap file per specified convention                                                                                                                                |
+| `python sample_app_util.py zap <zap_file> --generate-metadata [output_path]` | Generates the metadata file adjacent to the zap file with `.yaml` extension. If `[output_path]` is provided then the metadata file will be stored at the location specified. |
+
+## Running Tests
+
+---
+
+Navigate to the base directory of this README.
+
+```
+cd <project_root>/examples/chef/sample_app_util
+```
+
+Run unit tests.
+
+```
+python -m unittest
+```

examples/chef/sample_app_util/matter_device_types.json

@@ -0,0 +1,43 @@
+{
+    "Root Node": 22,
+    "Power Source": 17,
+    "OTA Requestor": 18,
+    "OTA Provider": 20,
+    "Aggregator": 14,
+    "Bridged Node": 19,
+    "On/Off Light": 256,
+    "Dimmable Light": 257,
+    "Color Temperature Light": 268,
+    "Extended Color Light": 269,
+    "On/Off Plug-in Unit": 266,
+    "Dimmable Plug-In Unit": 267,
+    "Pump": 771,
+    "On/Off Light Switch": 259,
+    "Dimmer Switch": 260,
+    "Color Dimmer Switch": 261,
+    "Control Bridge": 2112,
+    "Pump Controller": 772,
+    "Generic Switch": 15,
+    "Contact Sensor": 21,
+    "Light Sensor": 262,
+    "Occupancy Sensor": 263,
+    "Temperature Sensor": 770,
+    "Pressure Sensor": 773,
+    "Flow Sensor": 774,
+    "Humidity Sensor": 775,
+    "On/Off Sensor": 2128,
+    "Door Lock": 10,
+    "Door Lock Controller": 11,
+    "Window Covering": 514,
+    "Window Covering Controller": 515,
+    "Heating/Cooling Unit": 768,
+    "Thermostat": 769,
+    "Fan": 43,
+    "Casting Video Player": 35,
+    "Speaker": 34,
+    "Content App": 36,
+    "Basic Video Player": 40,
+    "Casting Video Client": 41,
+    "Video Remote Control": 42,
+    "Mode Select": 39
+}

examples/chef/sample_app_util/sample_app_util.py

@@ -0,0 +1,71 @@
+"""Utility to work with sample app device files.
+
+This utility helps with the following:
+- Parsing sample app device files.
+- Producing metadata files from sample app device files.
+- Generating names for sample app device files.
+
+Usage:
+  python sample_app_util.py zap <ZAP_FILE> [options]
+
+  python sample_app_util.py zap test_files/sample_zap_file.zap --generate-name
+  python sample_app_util.py zap test_files/sample_zap_file.zap --generate-metadata
+"""
+
+import argparse
+import os
+import shutil
+
+import zap_file_parser
+
+
+def zap_cmd_handler(args: argparse.Namespace) -> None:
+    """Handles args for zap_cmd_parser."""
+    zap_file_path = args.zap_file
+    if args.generate_name:
+        print(zap_file_parser.generate_name(zap_file_path))
+    elif args.rename_file:
+        name = zap_file_parser.generate_name(zap_file_path)
+        dirpath = os.path.dirname(zap_file_path)
+        hash_string = zap_file_parser.generate_hash(zap_file_path)
+        output_path = os.path.join(dirpath, f"{name}-{hash_string}.zap")
+        shutil.move(zap_file_path, output_path)
+        print(f"Renamed from: {zap_file_path} to {output_path}")
+    elif args.generate_hash_metadata:
+        created_file = zap_file_parser.generate_hash_metadata_file(zap_file_path)
+        print(f"Created {created_file}")
+
+
+parser = argparse.ArgumentParser()
+subparsers = parser.add_subparsers(dest="command")
+subparsers.required = True
+
+zap_cmd_parser = subparsers.add_parser("zap", help="Command to operate on zap files.")
+zap_cmd_parser.add_argument(
+    "zap_file", metavar="ZAP_FILE", type=str, help="Zap file to generate name for.")
+
+zap_cmd_group = zap_cmd_parser.add_mutually_exclusive_group()
+
+zap_cmd_group.add_argument(
+    "--generate-name", action="store_true",
+    help="Print the name file name according to the name convention"
+)
+
+zap_cmd_parser.add_argument(
+    "--generate-hash-metadata", action="store_true",
+    help=(
+        "Generate the hash metadata file which provide information about what was included in "
+        "the hash digest.")
+)
+
+zap_cmd_group.add_argument(
+    "--rename-file", action='store_true',
+    help="Rename the target zap file according to name convention."
+)
+
+zap_cmd_parser.set_defaults(func=zap_cmd_handler)
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    args.func(args)