Bring python yaml test parser of octet string inline with what javasc…

…ript codegen is doing (#23670)

* Fix octet string parsing in yaml test parser

Old parsing had issue since str.encode for utf-8 character would be
improperly convert. For example `\xff` would become `b'\xc3\xbf'` when
we wanted it to be `b'\xff`.

Co-authored-by: Tennessee Carmel-Veilleux <[email protected]>

* Address PR comments

* run regen

* Address PR comments

Co-authored-by: Tennessee Carmel-Veilleux <[email protected]>

src/app/tests/suites/TestCluster.yaml

@@ -749,7 +749,8 @@ tests:
       command: "readAttribute"
       attribute: "octet_string"
       response:
-          value: "\r\n\xff\"\xa0"
+          # This is the properly 'hex:...' version of "\r\n\xff\"\xa0"
+          value: "hex:0d0aff22a0"
 
     - label: "Write attribute OCTET_STRING"
       command: "writeAttribute"

src/controller/python/chip/yaml/format_converter.py

@@ -20,9 +20,26 @@
 from chip.tlv import uint, float32
 import enum
 from chip.yaml.errors import ValidationError
+import binascii
 
 
-_HEX_PREFIX = 'hex:'
+def convert_yaml_octet_string_to_bytes(s: str) -> bytes:
+    """Convert YAML octet string body to bytes, handling any c-style hex escapes (e.g. \x5a) and hex: prefix"""
+    # Step 1: handle explicit "hex:" prefix
+    if s.startswith('hex:'):
+        return binascii.unhexlify(s[4:])
+
+    # Step 2: convert non-hex-prefixed to bytes
+    # TODO(#23669): This does not properly support utf8 octet strings. We mimic
+    # javascript codegen behavior. Behavior of javascript is:
+    #   * Octet string character >= u+0200 errors out.
+    #   * Any character greater than 0xFF has the upper bytes chopped off.
+    as_bytes = [ord(c) for c in s]
+
+    if any([value > 0x200 for value in as_bytes]):
+        raise ValueError('Unsupported char in octet string %r' % as_bytes)
+    accumulated_hex = ''.join([f"{(v & 0xFF):02x}" for v in as_bytes])
+    return binascii.unhexlify(accumulated_hex)
 
 
 def convert_name_value_pair_to_dict(arg_values):
@@ -118,9 +135,7 @@ def convert_yaml_type(field_value, field_type, use_from_dict=False):
         return field_type(field_value)
     # YAML treats bytes as strings. Convert to a byte string.
     elif (field_type == bytes and type(field_value) != bytes):
-        if isinstance(field_value, str) and field_value.startswith(_HEX_PREFIX):
-            return bytes.fromhex(field_value[len(_HEX_PREFIX):])
-        return str.encode(field_value)
+        return convert_yaml_octet_string_to_bytes(field_value)
     # By default, just return the field_value casted to field_type.
     else:
         return field_type(field_value)

src/controller/python/test/unit_tests/test_yaml_format_converter.py

@@ -0,0 +1,52 @@
+#
+#    Copyright (c) 2022 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from chip.yaml.format_converter import convert_yaml_octet_string_to_bytes
+from binascii import unhexlify
+import unittest
+
+
+class TestOctetStringYamlDecode(unittest.TestCase):
+    def test_common_cases(self):
+        self.assertEqual(convert_yaml_octet_string_to_bytes("hex:aa55"), unhexlify("aa55"))
+        self.assertEqual(convert_yaml_octet_string_to_bytes("hex:"), unhexlify(""))
+        self.assertEqual(convert_yaml_octet_string_to_bytes("hex:AA55"), unhexlify("aa55"))
+
+        self.assertEqual(convert_yaml_octet_string_to_bytes("0\xaa\x55"), unhexlify("30aa55"))
+        self.assertEqual(convert_yaml_octet_string_to_bytes("0\xAA\x55"), unhexlify("30aa55"))
+        self.assertEqual(convert_yaml_octet_string_to_bytes("0\xAa\x55"), unhexlify("30aa55"))
+
+        self.assertEqual(convert_yaml_octet_string_to_bytes("0hex:"), b"0hex:")
+        self.assertEqual(convert_yaml_octet_string_to_bytes("0hex:A"), b"0hex:A")
+        self.assertEqual(convert_yaml_octet_string_to_bytes("0hex:AA55"), b"0hex:AA55")
+
+        self.assertEqual(convert_yaml_octet_string_to_bytes("AA55"), b"AA55")
+        self.assertEqual(convert_yaml_octet_string_to_bytes("AA\n\r\t55"), unhexlify("41410a0d093535"))
+        # TODO(#23669): After utf8 is properly supported expected result is unhexlify("c3a9c3a90a0a")
+        self.assertEqual(convert_yaml_octet_string_to_bytes("\xC3\xA9é\n\n"), unhexlify("c3a9e90a0a"))
+
+        # Partial hex nibble
+        with self.assertRaises(ValueError):
+            convert_yaml_octet_string_to_bytes("hex:aa5")
+
+
+def main():
+    unittest.main()
+
+
+if __name__ == "__main__":
+    main()