TC-IDM-XX: try both PASE and CASE for basic comp tests.

.github/workflows/tests.yaml

@@ -507,8 +507,6 @@ jobs:
                   mkdir -p out/trace_data
                   scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/controller/python/test/test_scripts/mobile-device-test.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/execute_python_tests.py --env-file /tmp/test_env.yaml --search-directory src/python_testing'
-                  scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TestMatterTestingSupport.py" --script-args "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
-                  scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TestSpecParsingSupport.py" --script-args "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
                   scripts/run_in_python_env.sh out/venv './scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py'
                   scripts/run_in_python_env.sh out/venv './src/python_testing/test_testing/test_TC_ICDM_2_1.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestIdChecks.py'
@@ -518,6 +516,8 @@ jobs:
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_IDM_10_4.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_TC_SC_7_1.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/TestDecorators.py'
+                  scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestMatterTestingSupport.py'
+                  scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestSpecParsingSupport.py'
 
 
             - name: Uploading core files

src/controller/python/ChipDeviceController-Discovery.cpp

@@ -23,12 +23,16 @@
  *
  */
 
+#include <string>
+
 #include <controller/CHIPDeviceController.h>
 #include <controller/python/chip/native/PyChipError.h>
 #include <json/json.h>
 #include <lib/core/CHIPError.h>
 #include <lib/core/TLV.h>
 #include <lib/dnssd/Resolver.h>
+#include <setup_payload/ManualSetupPayloadGenerator.h>
+#include <setup_payload/SetupPayload.h>
 
 using namespace chip;
 
@@ -186,4 +190,23 @@ bool pychip_DeviceController_GetIPForDiscoveredDevice(Controller::DeviceCommissi
     }
     return false;
 }
+
+PyChipError pychip_CreateManualCode(uint16_t longDiscriminator, uint32_t passcode, char * manualCodeBuffer, size_t bufsize)
+{
+    SetupPayload payload;
+    SetupDiscriminator discriminator;
+    discriminator.SetLongValue(longDiscriminator);
+    payload.discriminator = discriminator;
+    payload.setUpPINCode  = passcode;
+    std::string setupManualCode;
+
+    CHIP_ERROR err = ManualSetupPayloadGenerator(payload).payloadDecimalStringRepresentation(setupManualCode);
+    if (err == CHIP_NO_ERROR)
+    {
+        MutableCharSpan span(manualCodeBuffer, bufsize);
+        CopyCharSpanToMutableCharSpan(CharSpan(setupManualCode.c_str(), setupManualCode.length()), span);
+    }
+
+    return ToPyChipError(err);
+}
 }

src/controller/python/chip/ChipDeviceCtrl.py

@@ -1711,6 +1711,16 @@ def InitGroupTestingData(self):
                 self.devCtrl)
         ).raise_on_error()
 
+    def CreateManualCode(self, discriminator: int, passcode: int) -> str:
+        """ Creates a standard flow manual code from the given discriminator and passcode."""
+        # 64 bytes is WAY more than required, but let's be safe
+        size = 64
+        buf = create_string_buffer(size)
+        self._ChipStack.Call(
+            lambda: self._dmLib.pychip_CreateManualCode(discriminator, passcode, buf, size)
+        ).raise_on_error()
+        return buf.value.decode()
+
     # ----- Private Members -----
     def _InitLib(self):
         if self._dmLib is None:
@@ -1938,6 +1948,9 @@ def _InitLib(self):
             self._dmLib.pychip_DeviceProxy_GetRemoteSessionParameters.restype = PyChipError
             self._dmLib.pychip_DeviceProxy_GetRemoteSessionParameters.argtypes = [c_void_p, c_char_p]
 
+            self._dmLib.pychip_CreateManualCode.restype = PyChipError
+            self._dmLib.pychip_CreateManualCode.argtypes = [c_uint16, c_uint32, c_char_p, c_size_t]
+
 
 class ChipDeviceController(ChipDeviceControllerBase):
     ''' The ChipDeviceCommissioner binding, named as ChipDeviceController

src/python_testing/TC_DeviceBasicComposition.py

@@ -19,14 +19,58 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
+# test-runner-runs: run1 run2 run3 run4 run5 run6 run7
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --manual-code 10054912339 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+#
+# test-runner-run/run2/app: ${CHIP_LOCK_APP}
+# test-runner-run/run2/factoryreset: True
+# test-runner-run/run2/quiet: True
+# test-runner-run/run2/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run2/script-args: --storage-path admin_storage.json --manual-code 10054912339
+#
+# test-runner-run/run3/app: ${CHIP_LOCK_APP}
+# test-runner-run/run3/factoryreset: True
+# test-runner-run/run3/quiet: True
+# test-runner-run/run3/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run3/script-args: --storage-path admin_storage.json --qr-code MT:-24J0Q1212-10648G00
+#
+# test-runner-run/run4/app: ${CHIP_LOCK_APP}
+# test-runner-run/run4/factoryreset: True
+# test-runner-run/run4/quiet: True
+# test-runner-run/run4/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run4/script-args: --storage-path admin_storage.json --discriminator 1234 --passcode 20202021
+#
+# test-runner-run/run5/app: ${CHIP_LOCK_APP}
+# test-runner-run/run5/factoryreset: True
+# test-runner-run/run5/quiet: True
+# test-runner-run/run5/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run5/script-args: --storage-path admin_storage.json --manual-code 10054912339 --commissioning-method on-network
+#
+# test-runner-run/run6/app: ${CHIP_LOCK_APP}
+# test-runner-run/run6/factoryreset: True
+# test-runner-run/run6/quiet: True
+# test-runner-run/run6/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run6/script-args: --storage-path admin_storage.json --qr-code MT:-24J0Q1212-10648G00 --commissioning-method on-network
+#
+# test-runner-run/run7/app: ${CHIP_LOCK_APP}
+# test-runner-run/run7/factoryreset: True
+# test-runner-run/run7/quiet: True
+# test-runner-run/run7/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run7/script-args: --storage-path admin_storage.json --discriminator 1234 --passcode 20202021 --commissioning-method on-network
 # === END CI TEST ARGUMENTS ===
 
+# Run 1: runs through all tests
+# Run 2: tests PASE connection using manual code (12.1 only)
+# Run 3: tests PASE connection using QR code (12.1 only)
+# Run 4: tests PASE connection using discriminator and passcode (12.1 only)
+# Run 5: Tests CASE connection using manual code (12.1 only)
+# Run 6: Tests CASE connection using QR code (12.1 only)
+# Run 7: Tests CASE connection using manual discriminator and passcode (12.1 only)
+
 import logging
 from dataclasses import dataclass
 from typing import Any, Callable

src/python_testing/basic_composition_support.py

@@ -15,7 +15,7 @@
 #    limitations under the License.
 #
 
-
+import asyncio
 import base64
 import copy
 import json
@@ -98,12 +98,15 @@ def ConvertValue(value) -> Any:
 
 
 class BasicCompositionTests:
-    async def connect_over_pase(self, dev_ctrl):
-        asserts.assert_true(self.matter_test_config.qr_code_content == [] or self.matter_test_config.manual_code == [],
-                            "Cannot have both QR and manual code specified")
-        setupCode = self.matter_test_config.qr_code_content + self.matter_test_config.manual_code
-        asserts.assert_equal(len(setupCode), 1, "Require one of either --qr-code or --manual-code.")
-        await dev_ctrl.FindOrEstablishPASESession(setupCode[0], self.dut_node_id)
+    def get_code(self, dev_ctrl):
+        created_codes = []
+        for idx, d in enumerate(self.matter_test_config.discriminators):
+            created_codes.append(dev_ctrl.CreateManualCode(d, self.matter_test_config.setup_passcodes[idx]))
+
+        setup_codes = self.matter_test_config.qr_code_content + self.matter_test_config.manual_code + created_codes
+        asserts.assert_equal(len(setup_codes), 1,
+                             "Require exactly one of either --qr-code, --manual-code or (--discriminator and --passcode).")
+        return setup_codes[0]
 
     def dump_wildcard(self, dump_device_composition_path: typing.Optional[str]):
         node_dump_dict = {endpoint_id: MatterTlvToJson(self.endpoints_tlv[endpoint_id]) for endpoint_id in self.endpoints_tlv}
@@ -115,19 +118,34 @@ def dump_wildcard(self, dump_device_composition_path: typing.Optional[str]):
             with open(pathlib.Path(dump_device_composition_path).with_suffix(".txt"), "wt+") as outfile:
                 pprint(self.endpoints, outfile, indent=1, width=200, compact=True)
 
-    async def setup_class_helper(self, default_to_pase: bool = True):
+    async def setup_class_helper(self, allow_pase: bool = True):
         dev_ctrl = self.default_controller
         self.problems = []
 
-        do_test_over_pase = self.user_params.get("use_pase_only", default_to_pase)
         dump_device_composition_path: Optional[str] = self.user_params.get("dump_device_composition_path", None)
 
-        if do_test_over_pase:
-            await self.connect_over_pase(dev_ctrl)
-            node_id = self.dut_node_id
-        else:
-            # Using the already commissioned node
-            node_id = self.dut_node_id
+        node_id = self.dut_node_id
+
+        task_list = []
+        if allow_pase:
+            setup_code = self.get_code(dev_ctrl)
+            pase_future = dev_ctrl.EstablishPASESession(setup_code, self.dut_node_id)
+            task_list.append(asyncio.create_task(pase_future))
+
+        case_future = dev_ctrl.GetConnectedDevice(nodeid=node_id, allowPASE=False)
+        task_list.append(asyncio.create_task(case_future))
+
+        for task in task_list:
+            asyncio.ensure_future(task)
+
+        done, pending = await asyncio.wait(task_list, return_when=asyncio.FIRST_COMPLETED)
+
+        for task in pending:
+            try:
+                task.cancel()
+                await task
+            except asyncio.CancelledError:
+                pass
 
         wildcard_read = (await dev_ctrl.Read(node_id, [()]))
 

src/python_testing/matter_testing_support.py

@@ -510,8 +510,8 @@ class MatterTestConfig:
     app_pid: int = 0
 
     commissioning_method: Optional[str] = None
-    discriminators: Optional[List[int]] = None
-    setup_passcodes: Optional[List[int]] = None
+    discriminators: List[int] = field(default_factory=list)
+    setup_passcodes: List[int] = field(default_factory=list)
     commissionee_ip_address_just_for_testing: Optional[str] = None
     # By default, we start with maximized cert chains, as required for RR-1.1.
     # This allows cert tests to be run without re-commissioning for RR-1.1.
@@ -527,7 +527,7 @@ class MatterTestConfig:
     pics: dict[bool, str] = field(default_factory=dict)
 
     # Node ID for basic DUT
-    dut_node_ids: Optional[List[int]] = None
+    dut_node_ids: List[int] = field(default_factory=list)
     # Node ID to use for controller/commissioner
     controller_node_id: int = _DEFAULT_CONTROLLER_NODE_ID
     # CAT Tags for default controller/commissioner
@@ -1545,19 +1545,8 @@ def populate_commissioning_args(args: argparse.Namespace, config: MatterTestConf
 
     config.qr_code_content.extend(args.qr_code)
     config.manual_code.extend(args.manual_code)
-
-    if args.commissioning_method is None:
-        return True
-
-    if args.discriminators == [] and (args.qr_code == [] and args.manual_code == []):
-        print("error: Missing --discriminator when no --qr-code/--manual-code present!")
-        return False
-    config.discriminators = args.discriminators
-
-    if args.passcodes == [] and (args.qr_code == [] and args.manual_code == []):
-        print("error: Missing --passcode when no --qr-code/--manual-code present!")
-        return False
-    config.setup_passcodes = args.passcodes
+    config.discriminators.extend(args.discriminators)
+    config.setup_passcodes.extend(args.passcodes)
 
     if args.qr_code != [] and args.manual_code != []:
         print("error: Cannot have both --qr-code and --manual-code present!")
@@ -1574,9 +1563,11 @@ def populate_commissioning_args(args: argparse.Namespace, config: MatterTestConf
         return False
 
     if len(config.dut_node_ids) < len(device_descriptors):
-        missing = len(device_descriptors) - len(config.dut_node_ids)
         # We generate new node IDs sequentially from the last one seen for all
         # missing NodeIDs when commissioning many nodes at once.
+        if not config.dut_node_ids:
+            config.dut_node_ids = [_DEFAULT_DUT_NODE_ID]
+        missing = len(device_descriptors) - len(config.dut_node_ids)
         for i in range(missing):
             config.dut_node_ids.append(config.dut_node_ids[-1] + 1)
 
@@ -1588,6 +1579,17 @@ def populate_commissioning_args(args: argparse.Namespace, config: MatterTestConf
         print("error: Duplicate value in discriminator list")
         return False
 
+    if args.commissioning_method is None:
+        return True
+
+    if args.discriminators == [] and (args.qr_code == [] and args.manual_code == []):
+        print("error: Missing --discriminator when no --qr-code/--manual-code present!")
+        return False
+
+    if args.passcodes == [] and (args.qr_code == [] and args.manual_code == []):
+        print("error: Missing --passcode when no --qr-code/--manual-code present!")
+        return False
+
     if config.commissioning_method == "ble-wifi":
         if args.wifi_ssid is None:
             print("error: missing --wifi-ssid <SSID> for --commissioning-method ble-wifi!")
@@ -1684,7 +1686,7 @@ def parse_matter_test_args(argv: Optional[List[str]] = None) -> MatterTestConfig
                              default=_DEFAULT_CONTROLLER_NODE_ID,
                              help='NodeID to use for initial/default controller (default: %d)' % _DEFAULT_CONTROLLER_NODE_ID)
     basic_group.add_argument('-n', '--dut-node-id', '--nodeId', type=int_decimal_or_hex,
-                             metavar='NODE_ID', dest='dut_node_ids', default=[_DEFAULT_DUT_NODE_ID],
+                             metavar='NODE_ID', dest='dut_node_ids', default=[],
                              help='Node ID for primary DUT communication, '
                              'and NodeID to assign if commissioning (default: %d)' % _DEFAULT_DUT_NODE_ID, nargs="+")
     basic_group.add_argument('--endpoint', type=int, default=0, help="Endpoint under test")