Merge branch 'master' into feature/fix-tv-app-install-flow-and-suppor…

…ted-clusters

.github/workflows/build.yaml

@@ -332,7 +332,7 @@ jobs:
                   scripts/run_in_build_env.sh 'scripts/examples/gn_build_example.sh examples/chip-tool out/'
                   scripts/run_in_build_env.sh 'virtualenv pyenv'
                   source pyenv/bin/activate
-                  pip3 install -r src/setup_payload/python/requirements.txt
+                  pip3 install -r scripts/setup/requirements.setuppayload.txt
                   python3 src/setup_payload/tests/run_python_setup_payload_test.py out/chip-tool
 
     build_linux_python_lighting_device:

integrations/docker/images/base/chip-build/version

@@ -1 +1 @@
-60 : Add missing xz for esp32, upfate location of qemu-system-xtensa
+61 : Update env variable in vscode image

integrations/docker/images/vscode/chip-build-vscode/Dockerfile

@@ -129,7 +129,7 @@ ENV NXP_K32W1_SDK_ROOT=/opt/k32w/k32w1
 ENV NXP_SDK_ROOT=/opt/nxp-sdk/rw61x
 ENV OPENOCD_PATH=/opt/openocd/
 ENV PW_ENVIRONMENT_ROOT=/home/vscode/pigweed/env
-ENV QEMU_ESP32=/opt/espressif/qemu/xtensa-softmmu/qemu-system-xtensa
+ENV QEMU_ESP32=/opt/espressif/qemu/qemu-system-xtensa
 ENV QEMU_ESP32_DIR=/opt/espressif/qemu
 ENV SYSROOT_AARCH64=/opt/ubuntu-22.04.1-aarch64-sysroot
 ENV TELINK_ZEPHYR_BASE=/opt/telink/zephyrproject/zephyr

scripts/setup/requirements.nrfconnect.txt

@@ -1,6 +1,7 @@
+-r requirements.setuppayload.txt
+
 jsonschema>=4.4.0
 cbor2>=5.4.3
 ecdsa>=0.18.0
 qrcode==7.4.2
-bitarray==2.6.0
 python_stdnum==1.18

src/app/clusters/on-off-server/on-off-server.cpp

@@ -622,12 +622,7 @@ bool OnOffServer::offWithEffectCommand(app::CommandHandler * commandObj, const a
         if (globalSceneControl)
         {
 #ifdef MATTER_DM_PLUGIN_SCENES_MANAGEMENT
-            GroupId groupId = ZCL_SCENES_GLOBAL_SCENE_GROUP_ID;
-            if (commandObj->GetExchangeContext()->IsGroupExchangeContext())
-            {
-                groupId = commandObj->GetExchangeContext()->GetSessionHandle()->AsIncomingGroupSession()->GetGroupId();
-            }
-            ScenesManagement::ScenesServer::Instance().StoreCurrentScene(fabric, endpoint, groupId,
+            ScenesManagement::ScenesServer::Instance().StoreCurrentScene(fabric, endpoint, ZCL_SCENES_GLOBAL_SCENE_GROUP_ID,
                                                                          ZCL_SCENES_GLOBAL_SCENE_SCENE_ID);
 #endif // MATTER_DM_PLUGIN_SCENES_MANAGEMENT
             OnOff::Attributes::GlobalSceneControl::Set(endpoint, false);
@@ -683,13 +678,8 @@ bool OnOffServer::OnWithRecallGlobalSceneCommand(app::CommandHandler * commandOb
     }
 
 #ifdef MATTER_DM_PLUGIN_SCENES_MANAGEMENT
-    GroupId groupId = ZCL_SCENES_GLOBAL_SCENE_GROUP_ID;
-    if (commandObj->GetExchangeContext()->IsGroupExchangeContext())
-    {
-        groupId = commandObj->GetExchangeContext()->GetSessionHandle()->AsIncomingGroupSession()->GetGroupId();
-    }
-
-    ScenesManagement::ScenesServer::Instance().RecallScene(fabric, endpoint, groupId, ZCL_SCENES_GLOBAL_SCENE_SCENE_ID);
+    ScenesManagement::ScenesServer::Instance().RecallScene(fabric, endpoint, ZCL_SCENES_GLOBAL_SCENE_GROUP_ID,
+                                                           ZCL_SCENES_GLOBAL_SCENE_SCENE_ID);
 #endif // MATTER_DM_PLUGIN_SCENES_MANAGEMENT
 
     OnOff::Attributes::GlobalSceneControl::Set(endpoint, true);

src/controller/python/chip/ChipDeviceCtrl.py

@@ -38,7 +38,6 @@
 import logging
 import secrets
 import threading
-import time
 import typing
 from ctypes import (CDLL, CFUNCTYPE, POINTER, Structure, byref, c_bool, c_char, c_char_p, c_int, c_int32, c_size_t, c_uint8,
                     c_uint16, c_uint32, c_uint64, c_void_p, create_string_buffer, pointer, py_object, resize, string_at)
@@ -729,8 +728,8 @@ def GetAddressAndPort(self, nodeid):
 
         return (address.value.decode(), port.value) if error == 0 else None
 
-    def DiscoverCommissionableNodes(self, filterType: discovery.FilterType = discovery.FilterType.NONE, filter: typing.Any = None,
-                                    stopOnFirst: bool = False, timeoutSecond: int = 5) -> typing.Union[None, CommissionableNode, typing.List[CommissionableNode]]:
+    async def DiscoverCommissionableNodes(self, filterType: discovery.FilterType = discovery.FilterType.NONE, filter: typing.Any = None,
+                                          stopOnFirst: bool = False, timeoutSecond: int = 5) -> typing.Union[None, CommissionableNode, typing.List[CommissionableNode]]:
         ''' Discover commissionable nodes via DNS-SD with specified filters.
             Supported filters are:
 
@@ -752,27 +751,36 @@ def DiscoverCommissionableNodes(self, filterType: discovery.FilterType = discove
         if isinstance(filter, int):
             filter = str(filter)
 
-        self._ChipStack.Call(
-            lambda: self._dmLib.pychip_DeviceController_DiscoverCommissionableNodes(
-                self.devCtrl, int(filterType), str(filter).encode("utf-8"))).raise_on_error()
-
-        if timeoutSecond != 0:
-            if stopOnFirst:
-                target = time.time() + timeoutSecond
-                while time.time() < target:
-                    if self._ChipStack.Call(
-                            lambda: self._dmLib.pychip_DeviceController_HasDiscoveredCommissionableNode(self.devCtrl)):
-                        break
-                    time.sleep(0.1)
-            else:
-                time.sleep(timeoutSecond)
-
-        self._ChipStack.Call(
-            lambda: self._dmLib.pychip_DeviceController_StopCommissionableDiscovery(self.devCtrl)).raise_on_error()
+        # Discovery is also used during commissioning. Make sure this manual discovery
+        # and commissioning attempts do not interfere with each other.
+        async with self._commissioning_lock:
+            res = await self._ChipStack.CallAsync(
+                lambda: self._dmLib.pychip_DeviceController_DiscoverCommissionableNodes(
+                    self.devCtrl, int(filterType), str(filter).encode("utf-8")))
+            res.raise_on_error()
 
-        return self.GetDiscoveredDevices()
+            async def _wait_discovery():
+                while not await self._ChipStack.CallAsync(
+                        lambda: self._dmLib.pychip_DeviceController_HasDiscoveredCommissionableNode(self.devCtrl)):
+                    await asyncio.sleep(0.1)
+                return
 
-    def GetDiscoveredDevices(self):
+            try:
+                if stopOnFirst:
+                    await asyncio.wait_for(_wait_discovery(), timeoutSecond)
+                else:
+                    await asyncio.sleep(timeoutSecond)
+            except TimeoutError:
+                # Expected timeout, do nothing
+                pass
+            finally:
+                res = await self._ChipStack.CallAsync(
+                    lambda: self._dmLib.pychip_DeviceController_StopCommissionableDiscovery(self.devCtrl))
+                res.raise_on_error()
+
+            return await self.GetDiscoveredDevices()
+
+    async def GetDiscoveredDevices(self):
         def GetDevices(devCtrl):
             devices = []
 
@@ -786,7 +794,7 @@ def HandleDevice(deviceJson, deviceJsonLen):
             self._dmLib.pychip_DeviceController_IterateDiscoveredCommissionableNodes(devCtrl.devCtrl, HandleDevice)
             return devices
 
-        return self._ChipStack.Call(lambda: GetDevices(self))
+        return await self._ChipStack.CallAsync(lambda: GetDevices(self))
 
     def GetIPForDiscoveredDevice(self, idx, addrStr, length):
         self.CheckIsActive()

src/controller/python/chip/commissioning/pase.py

@@ -48,8 +48,8 @@ async def establish_session(devCtrl: ChipDeviceCtrl.ChipDeviceControllerBase, pa
     if isinstance(parameter, commissioning.PaseOverBLEParameters):
         await devCtrl.EstablishPASESessionBLE(parameter.setup_pin, parameter.discriminator, parameter.temporary_nodeid)
     elif isinstance(parameter, commissioning.PaseOverIPParameters):
-        device = devCtrl.DiscoverCommissionableNodes(filterType=discovery.FilterType.LONG_DISCRIMINATOR,
-                                                     filter=parameter.long_discriminator, stopOnFirst=True)
+        device = await devCtrl.DiscoverCommissionableNodes(filterType=discovery.FilterType.LONG_DISCRIMINATOR,
+                                                           filter=parameter.long_discriminator, stopOnFirst=True)
         if not device:
             raise ValueError("No commissionable device found")
         selected_address = None

src/controller/python/chip/yaml/runner.py

@@ -712,7 +712,7 @@ def __init__(self, test_step):
         self.filterType, self.filter = DiscoveryCommandAction._filter_for_step(test_step)
 
     async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult:
-        devices = dev_ctrl.DiscoverCommissionableNodes(
+        devices = await dev_ctrl.DiscoverCommissionableNodes(
             filterType=self.filterType, filter=self.filter, stopOnFirst=True, timeoutSecond=5)
 
         # Devices will be a list: [CommissionableNode(), ...]

src/controller/python/test/test_scripts/base.py

@@ -210,10 +210,10 @@ def _WaitForOneDiscoveredDevice(self, timeoutSeconds: int = 2):
             return None
         return ctypes.string_at(addrStrStorage).decode("utf-8")
 
-    def TestDiscovery(self, discriminator: int):
+    async def TestDiscovery(self, discriminator: int):
         self.logger.info(
             f"Discovering commissionable nodes with discriminator {discriminator}")
-        res = self.devCtrl.DiscoverCommissionableNodes(
+        res = await self.devCtrl.DiscoverCommissionableNodes(
             chip.discovery.FilterType.LONG_DISCRIMINATOR, discriminator, stopOnFirst=True, timeoutSecond=3)
         if not res:
             self.logger.info(
@@ -337,7 +337,7 @@ async def TestCommissioningWithSetupPayload(self, setupPayload: str, nodeid: int
 
     async def TestOnNetworkCommissioning(self, discriminator: int, setuppin: int, nodeid: int, ip_override: str = None):
         self.logger.info("Testing discovery")
-        device = self.TestDiscovery(discriminator=discriminator)
+        device = await self.TestDiscovery(discriminator=discriminator)
         if not device:
             self.logger.info("Failed to discover any devices.")
             return False

src/controller/python/test/test_scripts/commissioning_test.py

@@ -125,7 +125,7 @@ async def main():
         nodeid=112233, paaTrustStorePath=options.paaTrustStorePath, testCommissioner=True)
 
     logger.info("Testing discovery")
-    FailIfNot(test.TestDiscovery(discriminator=options.discriminator),
+    FailIfNot(await test.TestDiscovery(discriminator=options.discriminator),
               "Failed to discover any devices.")
 
     FailIfNot(test.SetNetworkCommissioningParameters(dataset=TEST_THREAD_NETWORK_DATASET_TLV),

src/controller/python/test/test_scripts/failsafe_tests.py

@@ -88,7 +88,7 @@ async def main():
         nodeid=112233, paaTrustStorePath=options.paaTrustStorePath, testCommissioner=False)
 
     logger.info("Testing discovery")
-    FailIfNot(test.TestDiscovery(discriminator=TEST_DISCRIMINATOR),
+    FailIfNot(await test.TestDiscovery(discriminator=TEST_DISCRIMINATOR),
               "Failed to discover any devices.")
 
     FailIfNot(test.SetNetworkCommissioningParameters(dataset=TEST_THREAD_NETWORK_DATASET_TLV),

src/controller/python/test/test_scripts/mobile-device-test.py

@@ -59,7 +59,7 @@
 
 async def ethernet_commissioning(test: BaseTestHelper, discriminator: int, setup_pin: int, address_override: str, device_nodeid: int):
     logger.info("Testing discovery")
-    device = test.TestDiscovery(discriminator=discriminator)
+    device = await test.TestDiscovery(discriminator=discriminator)
     FailIfNot(device, "Failed to discover any devices.")
 
     address = device.addresses[0]

src/python_testing/TC_IDM_1_2.py

@@ -202,7 +202,7 @@ async def test_TC_IDM_1_2(self):
         new_fabric_admin = new_certificate_authority.NewFabricAdmin(vendorId=0xFFF1, fabricId=self.matter_test_config.fabric_id + 1)
         TH2 = new_fabric_admin.NewController(nodeId=112233)
 
-        devices = TH2.DiscoverCommissionableNodes(
+        devices = await TH2.DiscoverCommissionableNodes(
             filterType=Discovery.FilterType.LONG_DISCRIMINATOR, filter=discriminator, stopOnFirst=False)
         # For some reason, the devices returned here aren't filtered, so filter ourselves
         device = next(filter(lambda d: d.commissioningMode == 2 and d.longDiscriminator == discriminator, devices))

src/python_testing/TC_OPCREDS_3_1.py

@@ -34,7 +34,7 @@ async def FindAndEstablishPase(self, longDiscriminator: int, setupPinCode: int,
         if dev_ctrl is None:
             dev_ctrl = self.default_controller
 
-        devices = dev_ctrl.DiscoverCommissionableNodes(
+        devices = await dev_ctrl.DiscoverCommissionableNodes(
             filterType=Discovery.FilterType.LONG_DISCRIMINATOR, filter=longDiscriminator, stopOnFirst=False)
         # For some reason, the devices returned here aren't filtered, so filter ourselves
         device = next(filter(lambda d: d.commissioningMode ==