TC-IDM-XX: try both PASE and CASE for basic comp tests. (#35109)

* TC-IDM-XX: try both PASE and CASE for basic comp tests.

These tests are disruptive at cert because they could run over
PASE or CASE and defaulted to PASE. This meant that they couldn't
be run in sequence with other tests that require commissioning.
Because the PASE setup required a qr or manual code, it also
meant that these needed special parameters to run.

Now:
- can use discriminator and passcode for PASE connection
- device tries both PASE and CASE and runs over whichever works
  first.

* fix CI

* Restyled by whitespace

* Restyled by clang-format

* Fix tests for NO DUT (unit and mock)

* Update src/python_testing/basic_composition_support.py

Co-authored-by: Tennessee Carmel-Veilleux <[email protected]>

* Null terminate and add outbuf size

* Restyled by clang-format

* typo

---------

Co-authored-by: Restyled.io <[email protected]>
Co-authored-by: Tennessee Carmel-Veilleux <[email protected]>

.github/workflows/tests.yaml

@@ -515,8 +515,6 @@ jobs:
                   mkdir -p out/trace_data
                   scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/controller/python/test/test_scripts/mobile-device-test.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/execute_python_tests.py --env-file /tmp/test_env.yaml --search-directory src/python_testing'
-                  scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TestMatterTestingSupport.py" --script-args "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
-                  scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TestSpecParsingSupport.py" --script-args "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"'
                   scripts/run_in_python_env.sh out/venv './scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py'
                   scripts/run_in_python_env.sh out/venv './src/python_testing/test_testing/test_TC_ICDM_2_1.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestIdChecks.py'
@@ -528,6 +526,8 @@ jobs:
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_IDM_10_4.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_TC_SC_7_1.py'
                   scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/TestDecorators.py'
+                  scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestMatterTestingSupport.py'
+                  scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestSpecParsingSupport.py'
 
 
             - name: Uploading core files

src/controller/python/ChipDeviceController-Discovery.cpp

@@ -23,12 +23,16 @@
  *
  */
 
+#include <string>
+
 #include <controller/CHIPDeviceController.h>
 #include <controller/python/chip/native/PyChipError.h>
 #include <json/json.h>
 #include <lib/core/CHIPError.h>
 #include <lib/core/TLV.h>
 #include <lib/dnssd/Resolver.h>
+#include <setup_payload/ManualSetupPayloadGenerator.h>
+#include <setup_payload/SetupPayload.h>
 
 using namespace chip;
 
@@ -186,4 +190,31 @@ bool pychip_DeviceController_GetIPForDiscoveredDevice(Controller::DeviceCommissi
     }
     return false;
 }
+
+PyChipError pychip_CreateManualCode(uint16_t longDiscriminator, uint32_t passcode, char * manualCodeBuffer, size_t inBufSize,
+                                    size_t * outBufSize)
+{
+    SetupPayload payload;
+    SetupDiscriminator discriminator;
+    discriminator.SetLongValue(longDiscriminator);
+    payload.discriminator = discriminator;
+    payload.setUpPINCode  = passcode;
+    std::string setupManualCode;
+
+    *outBufSize    = 0;
+    CHIP_ERROR err = ManualSetupPayloadGenerator(payload).payloadDecimalStringRepresentation(setupManualCode);
+    if (err == CHIP_NO_ERROR)
+    {
+        MutableCharSpan span(manualCodeBuffer, inBufSize);
+        // Plus 1 so we copy the null terminator
+        CopyCharSpanToMutableCharSpan(CharSpan(setupManualCode.c_str(), setupManualCode.length() + 1), span);
+        *outBufSize = span.size();
+        if (*outBufSize == 0)
+        {
+            err = CHIP_ERROR_NO_MEMORY;
+        }
+    }
+
+    return ToPyChipError(err);
+}
 }

src/controller/python/chip/ChipDeviceCtrl.py

@@ -1711,6 +1711,19 @@ def InitGroupTestingData(self):
                 self.devCtrl)
         ).raise_on_error()
 
+    def CreateManualCode(self, discriminator: int, passcode: int) -> str:
+        """ Creates a standard flow manual code from the given discriminator and passcode."""
+        # 64 bytes is WAY more than required, but let's be safe
+        in_size = 64
+        out_size = c_size_t(0)
+        buf = create_string_buffer(in_size)
+        self._ChipStack.Call(
+            lambda: self._dmLib.pychip_CreateManualCode(discriminator, passcode, buf, in_size, pointer(out_size))
+        ).raise_on_error()
+        if out_size.value == 0 or out_size.value > in_size:
+            raise MemoryError("Invalid output size for manual code")
+        return buf.value.decode()
+
     # ----- Private Members -----
     def _InitLib(self):
         if self._dmLib is None:
@@ -1938,6 +1951,9 @@ def _InitLib(self):
             self._dmLib.pychip_DeviceProxy_GetRemoteSessionParameters.restype = PyChipError
             self._dmLib.pychip_DeviceProxy_GetRemoteSessionParameters.argtypes = [c_void_p, c_char_p]
 
+            self._dmLib.pychip_CreateManualCode.restype = PyChipError
+            self._dmLib.pychip_CreateManualCode.argtypes = [c_uint16, c_uint32, c_char_p, c_size_t, POINTER(c_size_t)]
+
 
 class ChipDeviceController(ChipDeviceControllerBase):
     ''' The ChipDeviceCommissioner binding, named as ChipDeviceController

src/python_testing/TC_DeviceBasicComposition.py

@@ -19,14 +19,58 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
+# test-runner-runs: run1 run2 run3 run4 run5 run6 run7
 # test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
 # test-runner-run/run1/factoryreset: True
 # test-runner-run/run1/quiet: True
 # test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
 # test-runner-run/run1/script-args: --storage-path admin_storage.json --manual-code 10054912339 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+#
+# test-runner-run/run2/app: ${CHIP_LOCK_APP}
+# test-runner-run/run2/factoryreset: True
+# test-runner-run/run2/quiet: True
+# test-runner-run/run2/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run2/script-args: --storage-path admin_storage.json --manual-code 10054912339
+#
+# test-runner-run/run3/app: ${CHIP_LOCK_APP}
+# test-runner-run/run3/factoryreset: True
+# test-runner-run/run3/quiet: True
+# test-runner-run/run3/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run3/script-args: --storage-path admin_storage.json --qr-code MT:-24J0Q1212-10648G00
+#
+# test-runner-run/run4/app: ${CHIP_LOCK_APP}
+# test-runner-run/run4/factoryreset: True
+# test-runner-run/run4/quiet: True
+# test-runner-run/run4/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run4/script-args: --storage-path admin_storage.json --discriminator 1234 --passcode 20202021
+#
+# test-runner-run/run5/app: ${CHIP_LOCK_APP}
+# test-runner-run/run5/factoryreset: True
+# test-runner-run/run5/quiet: True
+# test-runner-run/run5/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run5/script-args: --storage-path admin_storage.json --manual-code 10054912339 --commissioning-method on-network
+#
+# test-runner-run/run6/app: ${CHIP_LOCK_APP}
+# test-runner-run/run6/factoryreset: True
+# test-runner-run/run6/quiet: True
+# test-runner-run/run6/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run6/script-args: --storage-path admin_storage.json --qr-code MT:-24J0Q1212-10648G00 --commissioning-method on-network
+#
+# test-runner-run/run7/app: ${CHIP_LOCK_APP}
+# test-runner-run/run7/factoryreset: True
+# test-runner-run/run7/quiet: True
+# test-runner-run/run7/app-args: --discriminator 1234 --KVS kvs1
+# test-runner-run/run7/script-args: --storage-path admin_storage.json --discriminator 1234 --passcode 20202021 --commissioning-method on-network
 # === END CI TEST ARGUMENTS ===
 
+# Run 1: runs through all tests
+# Run 2: tests PASE connection using manual code (12.1 only)
+# Run 3: tests PASE connection using QR code (12.1 only)
+# Run 4: tests PASE connection using discriminator and passcode (12.1 only)
+# Run 5: Tests CASE connection using manual code (12.1 only)
+# Run 6: Tests CASE connection using QR code (12.1 only)
+# Run 7: Tests CASE connection using manual discriminator and passcode (12.1 only)
+
 import logging
 from dataclasses import dataclass
 from typing import Any, Callable

src/python_testing/basic_composition_support.py

@@ -15,7 +15,7 @@
 #    limitations under the License.
 #
 
-
+import asyncio
 import base64
 import copy
 import json
@@ -98,12 +98,15 @@ def ConvertValue(value) -> Any:
 
 
 class BasicCompositionTests:
-    async def connect_over_pase(self, dev_ctrl):
-        asserts.assert_true(self.matter_test_config.qr_code_content == [] or self.matter_test_config.manual_code == [],
-                            "Cannot have both QR and manual code specified")
-        setupCode = self.matter_test_config.qr_code_content + self.matter_test_config.manual_code
-        asserts.assert_equal(len(setupCode), 1, "Require one of either --qr-code or --manual-code.")
-        await dev_ctrl.FindOrEstablishPASESession(setupCode[0], self.dut_node_id)
+    def get_code(self, dev_ctrl):
+        created_codes = []
+        for idx, discriminator in enumerate(self.matter_test_config.discriminators):
+            created_codes.append(dev_ctrl.CreateManualCode(discriminator, self.matter_test_config.setup_passcodes[idx]))
+
+        setup_codes = self.matter_test_config.qr_code_content + self.matter_test_config.manual_code + created_codes
+        asserts.assert_equal(len(setup_codes), 1,
+                             "Require exactly one of either --qr-code, --manual-code or (--discriminator and --passcode).")
+        return setup_codes[0]
 
     def dump_wildcard(self, dump_device_composition_path: typing.Optional[str]) -> tuple[str, str]:
         """ Dumps a json and a txt file of the attribute wildcard for this device if the dump_device_composition_path is supplied.
@@ -120,19 +123,34 @@ def dump_wildcard(self, dump_device_composition_path: typing.Optional[str]) -> t
                 pprint(self.endpoints, outfile, indent=1, width=200, compact=True)
         return (json_dump_string, pformat(self.endpoints, indent=1, width=200, compact=True))
 
-    async def setup_class_helper(self, default_to_pase: bool = True):
+    async def setup_class_helper(self, allow_pase: bool = True):
         dev_ctrl = self.default_controller
         self.problems = []
 
-        do_test_over_pase = self.user_params.get("use_pase_only", default_to_pase)
         dump_device_composition_path: Optional[str] = self.user_params.get("dump_device_composition_path", None)
 
-        if do_test_over_pase:
-            await self.connect_over_pase(dev_ctrl)
-            node_id = self.dut_node_id
-        else:
-            # Using the already commissioned node
-            node_id = self.dut_node_id
+        node_id = self.dut_node_id
+
+        task_list = []
+        if allow_pase:
+            setup_code = self.get_code(dev_ctrl)
+            pase_future = dev_ctrl.EstablishPASESession(setup_code, self.dut_node_id)
+            task_list.append(asyncio.create_task(pase_future))
+
+        case_future = dev_ctrl.GetConnectedDevice(nodeid=node_id, allowPASE=False)
+        task_list.append(asyncio.create_task(case_future))
+
+        for task in task_list:
+            asyncio.ensure_future(task)
+
+        done, pending = await asyncio.wait(task_list, return_when=asyncio.FIRST_COMPLETED)
+
+        for task in pending:
+            try:
+                task.cancel()
+                await task
+            except asyncio.CancelledError:
+                pass
 
         wildcard_read = (await dev_ctrl.Read(node_id, [()]))
 

src/python_testing/matter_testing_support.py

@@ -629,8 +629,8 @@ class MatterTestConfig:
     app_pid: int = 0
 
     commissioning_method: Optional[str] = None
-    discriminators: Optional[List[int]] = None
-    setup_passcodes: Optional[List[int]] = None
+    discriminators: List[int] = field(default_factory=list)
+    setup_passcodes: List[int] = field(default_factory=list)
     commissionee_ip_address_just_for_testing: Optional[str] = None
     # By default, we start with maximized cert chains, as required for RR-1.1.
     # This allows cert tests to be run without re-commissioning for RR-1.1.
@@ -646,7 +646,7 @@ class MatterTestConfig:
     pics: dict[bool, str] = field(default_factory=dict)
 
     # Node ID for basic DUT
-    dut_node_ids: Optional[List[int]] = None
+    dut_node_ids: List[int] = field(default_factory=list)
     # Node ID to use for controller/commissioner
     controller_node_id: int = _DEFAULT_CONTROLLER_NODE_ID
     # CAT Tags for default controller/commissioner
@@ -1730,19 +1730,8 @@ def populate_commissioning_args(args: argparse.Namespace, config: MatterTestConf
 
     config.qr_code_content.extend(args.qr_code)
     config.manual_code.extend(args.manual_code)
-
-    if args.commissioning_method is None:
-        return True
-
-    if args.discriminators == [] and (args.qr_code == [] and args.manual_code == []):
-        print("error: Missing --discriminator when no --qr-code/--manual-code present!")
-        return False
-    config.discriminators = args.discriminators
-
-    if args.passcodes == [] and (args.qr_code == [] and args.manual_code == []):
-        print("error: Missing --passcode when no --qr-code/--manual-code present!")
-        return False
-    config.setup_passcodes = args.passcodes
+    config.discriminators.extend(args.discriminators)
+    config.setup_passcodes.extend(args.passcodes)
 
     if args.qr_code != [] and args.manual_code != []:
         print("error: Cannot have both --qr-code and --manual-code present!")
@@ -1759,9 +1748,11 @@ def populate_commissioning_args(args: argparse.Namespace, config: MatterTestConf
         return False
 
     if len(config.dut_node_ids) < len(device_descriptors):
-        missing = len(device_descriptors) - len(config.dut_node_ids)
         # We generate new node IDs sequentially from the last one seen for all
         # missing NodeIDs when commissioning many nodes at once.
+        if not config.dut_node_ids:
+            config.dut_node_ids = [_DEFAULT_DUT_NODE_ID]
+        missing = len(device_descriptors) - len(config.dut_node_ids)
         for i in range(missing):
             config.dut_node_ids.append(config.dut_node_ids[-1] + 1)
 
@@ -1773,6 +1764,17 @@ def populate_commissioning_args(args: argparse.Namespace, config: MatterTestConf
         print("error: Duplicate value in discriminator list")
         return False
 
+    if args.commissioning_method is None:
+        return True
+
+    if args.discriminators == [] and (args.qr_code == [] and args.manual_code == []):
+        print("error: Missing --discriminator when no --qr-code/--manual-code present!")
+        return False
+
+    if args.passcodes == [] and (args.qr_code == [] and args.manual_code == []):
+        print("error: Missing --passcode when no --qr-code/--manual-code present!")
+        return False
+
     if config.commissioning_method == "ble-wifi":
         if args.wifi_ssid is None:
             print("error: missing --wifi-ssid <SSID> for --commissioning-method ble-wifi!")
@@ -1869,7 +1871,7 @@ def parse_matter_test_args(argv: Optional[List[str]] = None) -> MatterTestConfig
                              default=_DEFAULT_CONTROLLER_NODE_ID,
                              help='NodeID to use for initial/default controller (default: %d)' % _DEFAULT_CONTROLLER_NODE_ID)
     basic_group.add_argument('-n', '--dut-node-id', '--nodeId', type=int_decimal_or_hex,
-                             metavar='NODE_ID', dest='dut_node_ids', default=[_DEFAULT_DUT_NODE_ID],
+                             metavar='NODE_ID', dest='dut_node_ids', default=[],
                              help='Node ID for primary DUT communication, '
                              'and NodeID to assign if commissioning (default: %d)' % _DEFAULT_DUT_NODE_ID, nargs="+")
     basic_group.add_argument('--endpoint', type=int, default=0, help="Endpoint under test")