Add a parser for CSA XML data models

.github/workflows/tests.yaml

@@ -85,7 +85,7 @@ jobs:
               #
               run: |
                   ./scripts/run_in_build_env.sh \
-                    "./scripts/py_matter_idl/matter_idl/xml_parser.py \
+                    "./scripts/py_matter_idl/matter_idl/zapxml_parser.py \
                       --no-print \
                       --log-level info \
                       src/app/zap-templates/zcl/data-model/chip/global-attributes.xml \

scripts/py_matter_idl/BUILD.gn

@@ -67,7 +67,7 @@ pw_python_package("matter_idl") {
     "matter_idl/test_matter_idl_parser.py",
     "matter_idl/test_generators.py",
     "matter_idl/test_idl_generator.py",
-    "matter_idl/test_xml_parser.py",
+    "matter_idl/test_zapxml.py",
   ]
 
   # TODO: at a future time consider enabling all (* or missing) here to get

scripts/py_matter_idl/files.gni

@@ -23,6 +23,12 @@ matter_idl_generator_templates = [
 matter_idl_generator_sources = [
   "${chip_root}/scripts/py_matter_idl/matter_idl/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/backwards_compatibility.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/__init__.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/__init__.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/base.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/context.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/handlers.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/parsing.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/__init__.py",
@@ -38,16 +44,17 @@ matter_idl_generator_sources = [
   "${chip_root}/scripts/py_matter_idl/matter_idl/matter_idl_parser.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/matter_idl_types.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/test_backwards_compatibility.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/test_data_model_xml.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/test_generators.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/test_matter_idl_parser.py",
-  "${chip_root}/scripts/py_matter_idl/matter_idl/test_xml_parser.py",
-  "${chip_root}/scripts/py_matter_idl/matter_idl/xml_parser.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/test_zapxml.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/__init__.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/base.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/context.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/handlers.py",
   "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/parsing.py",
+  "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml_parser.py",
 ]
 
 # All the files that the matter idl infrastructure will use

scripts/py_matter_idl/matter_idl/data_model_xml/__init__.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import typing
+import xml.sax.handler
+from dataclasses import dataclass
+from typing import List, Optional, Union
+
+from matter_idl.data_model_xml.handlers import Context, DataModelXmlHandler
+from matter_idl.matter_idl_types import Idl
+
+
+class ParseHandler(xml.sax.handler.ContentHandler):
+    """A parser for data model XML data definitions.
+
+    Defers its processing to DataModelXmlHandler and keeps track of:
+       - an internal context for all handlers
+       - the parsed Idl structure that is incrementally built
+       - sets up parsing location within the context
+       - keeps track of ParsePath
+
+    Overall converts a python SAX handler into matter_idl.zapxml.handlers
+    """
+
+    def __init__(self, include_meta_data=True):
+        super().__init__()
+        self._idl = Idl()
+        self._processing_stack = []
+        # Context persists across all
+        self._context = Context()
+        self._include_meta_data = include_meta_data
+        self._locator = None
+
+    def PrepareParsing(self, filename):
+        # This is a bit ugly: filename keeps changing during parse
+        # IDL meta is not prepared for this (as source is XML and .matter is
+        # single file)
+        if self._include_meta_data:
+            self._idl.parse_file_name = filename
+
+        self._context.file_name = filename
+
+    def Finish(self) -> Idl:
+        self._context.PostProcess(self._idl)
+        return self._idl
+
+    def startDocument(self):
+        if self._include_meta_data and self._locator:
+            self._context.locator = self._locator
+        self._processing_stack = [
+            DataModelXmlHandler(self._context, self._idl)]
+
+    def endDocument(self):
+        if len(self._processing_stack) != 1:
+            raise Exception("Unexpected nesting!")
+
+    def startElement(self, name: str, attrs):
+        logging.debug("ELEMENT START: %r / %r" % (name, attrs))
+        self._context.path.push(name)
+        self._processing_stack.append(
+            self._processing_stack[-1].GetNextProcessor(name, attrs))
+
+    def endElement(self, name: str):
+        logging.debug("ELEMENT END: %r" % name)
+
+        last = self._processing_stack.pop()
+        last.EndProcessing()
+
+        # important to pop AFTER processing end to allow processing
+        # end to access the current context
+        self._context.path.pop()
+
+    def characters(self, content):
+        self._processing_stack[-1].HandleContent(content)
+
+
+@dataclass
+class ParseSource:
+    """Represents an input sopurce for ParseXmls.
+
+    Allows for named data sources to be parsed.
+    """
+    source: Union[str, typing.IO]  # filename or stream
+    # actual filename to use, None if the source is a filename already
+    name: Optional[str] = None
+
+    @ property
+    def source_file_name(self):
+        if self.name:
+            return self.name
+        return self.source  # assume string
+
+
+def ParseXmls(sources: List[ParseSource], include_meta_data=True) -> Idl:
+    """Parse one or more XML inputs and return the resulting Idl data.
+
+    Params:
+       sources - what to parse
+       include_meta_data - if parsing location data should be included in the Idl
+    """
+    handler = ParseHandler(include_meta_data=include_meta_data)
+
+    for source in sources:
+        logging.info('Parsing %s...' % source.source_file_name)
+        handler.PrepareParsing(source.source_file_name)
+
+        parser = xml.sax.make_parser()
+        parser.setContentHandler(handler)
+        try:
+            parser.parse(source.source)
+        except AssertionError as e:
+            logging.error("AssertionError %s at %r", e,
+                          handler._context.GetCurrentLocationMeta())
+            raise
+
+    return handler.Finish()

scripts/py_matter_idl/matter_idl/data_model_xml/handlers/__init__.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from matter_idl.matter_idl_types import Idl
+
+from .base import BaseHandler
+from .context import Context
+from .handlers import ClusterHandler
+
+
+class DataModelXmlHandler(BaseHandler):
+    """Handles the top level (/) of a data model xml file
+    """
+
+    def __init__(self, context: Context, idl: Idl):
+        super().__init__(context)
+        self._idl = idl
+
+    def GetNextProcessor(self, name, attrs):
+        if name.lower() == 'cluster':
+            return ClusterHandler(self.context, self._idl, attrs)
+        else:
+            return BaseHandler(self.context)

scripts/py_matter_idl/matter_idl/data_model_xml/handlers/base.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+
+from .context import Context
+
+
+class HandledDepth:
+    """Defines how deep a XML element has been handled."""
+    NOT_HANDLED = enum.auto()  # Unknown/parsed element
+    ENTIRE_TREE = enum.auto()  # Entire tree can be ignored
+    SINGLE_TAG = enum.auto()  # Single tag processed, but not sub-items
+
+
+class BaseHandler:
+    """A generic element handler.
+
+       XML processing is done in the form of depth-first processing:
+          - Tree is descended into using `GetNextProcessor`
+          - Processors are expected to extend `BaseHandler` and allow for:
+             - GetNextProcessor to recurse
+             - HandleContent in case the text content is relevant
+             - EndProcessing once the entire tree has been walked (when xml element ends)
+
+       BaseHandler keeps track if it has been handled or ot by its `_handled` setting and
+       init parameter. Non-handled elements will be tagged within the context, resulting
+       in logs. This is to detect if unknown/new tags appear in XML files.
+    """
+
+    def __init__(self, context: Context, handled=HandledDepth.NOT_HANDLED):
+        self.context = context
+        self._handled = handled
+
+    def GetNextProcessor(self, name, attrs):
+        """Get the next processor to use for the given name"""
+
+        if self._handled == HandledDepth.SINGLE_TAG:
+            handled = HandledDepth.NOT_HANDLED
+        else:
+            handled = self._handled
+
+        return BaseHandler(context=self.context, handled=handled)
+
+    def HandleContent(self, content):
+        """Processes some content"""
+        pass
+
+    def EndProcessing(self):
+        """Finalizes the processing of the current element"""
+        if self._handled == HandledDepth.NOT_HANDLED:
+            self.context.MarkTagNotHandled()