Introduce Python-based freestanding tests (#21567)

* Introduce Python-based freestanding tests

- Testing with Cirque requires complex setup and it is not easy to add
  tests.
- Testing with chip-repl is interactive only
- We needed a way to test more complex test scenarios than possible
  with chip-tool which cannot deal with complex test cases including recomputed
  cryptographic values due to complex logic and data dependencies.

This PR:

- Uses existing Mobly test framework dependency to add capabilities to
  make freestanding tests (see `src/python_testing/hello_test.py`)
- Adds a module that only depends on existing CHIP modules in the virtual
  environment to provide needed scaffolding for tests running, including
  being able to commission a device from the command line
- Adds needed cryptographic dependencies to environment for upcoming tests
- Adds an example test (`hello_test.py`)
- Added support to Python ChipDeviceCtrl to do on-network commissioning
  (thanks @erjiaqing!)

What will follow-up:
- Better docs
- More sample tests
- A way to re-use credentials from chip-tool with Python (already WIP)

Testing done:
- Built a test for TC-DA-1.7 (not included here) and ran it successfully against
  an all-clusters-app
- Unit tests still pass
- Integration tests still pass

* Restyled by clang-format

* Restyled by gn

* Restyled by autopep8

Co-authored-by: Restyled.io <[email protected]>

scripts/constraints.txt

@@ -153,7 +153,7 @@ mbed-ls==1.8.11 ; platform_machine != "aarch64" and sys_platform == "linux"
     # via -r requirements.mbed.txt
 mbed-os-tools==1.8.13
     # via mbed-ls
-mobly==1.10.1
+mobly==1.11.1
     # via -r requirements.txt
 numpy==1.23.0
     # via pandas

src/controller/python/BUILD.gn

@@ -55,6 +55,8 @@ shared_library("ChipDeviceCtrl") {
       "ChipDeviceController-ScriptBinding.cpp",
       "ChipDeviceController-ScriptDevicePairingDelegate.cpp",
       "ChipDeviceController-ScriptDevicePairingDelegate.h",
+      "ChipDeviceController-ScriptPairingDeviceDiscoveryDelegate.cpp",
+      "ChipDeviceController-ScriptPairingDeviceDiscoveryDelegate.h",
       "ChipDeviceController-StorageDelegate.cpp",
       "ChipDeviceController-StorageDelegate.h",
       "OpCredsBinding.cpp",
@@ -281,6 +283,12 @@ chip_python_wheel_action("chip-core") {
     "pyyaml",
     "ipdb",
     "deprecation",
+    "mobly",
+
+    # Crypto libraries for complex tests and internal Python controller usage
+    "cryptography",
+    "pycrypto",
+    "ecdsa",
   ]
 
   if (current_os == "mac") {
@@ -397,6 +405,7 @@ chip_python_wheel_action("chip-repl") {
     "ipython!=8.1.0",
     "rich",
     "ipykernel",
+    "mobly",
   ]
 
   py_package_name = "chip-repl"

src/controller/python/ChipDeviceController-ScriptBinding.cpp

@@ -51,6 +51,7 @@
 #include <controller/ExampleOperationalCredentialsIssuer.h>
 
 #include <controller/python/ChipDeviceController-ScriptDevicePairingDelegate.h>
+#include <controller/python/ChipDeviceController-ScriptPairingDeviceDiscoveryDelegate.h>
 #include <controller/python/ChipDeviceController-StorageDelegate.h>
 #include <controller/python/chip/interaction_model/Delegate.h>
 
@@ -98,6 +99,7 @@ chip::Controller::CommissioningParameters sCommissioningParameters;
 } // namespace
 
 chip::Controller::ScriptDevicePairingDelegate sPairingDelegate;
+chip::Controller::ScriptPairingDeviceDiscoveryDelegate sPairingDeviceDiscoveryDelegate;
 chip::Controller::Python::StorageAdapter * sStorageAdapter = nullptr;
 chip::Credentials::GroupDataProviderImpl sGroupDataProvider;
 chip::Credentials::PersistentStorageOpCertStore sPersistentStorageOpCertStore;
@@ -152,6 +154,11 @@ ChipError::StorageType pychip_DeviceController_DiscoverCommissionableNodesDevice
                                                                                      uint16_t device_type);
 ChipError::StorageType
 pychip_DeviceController_DiscoverCommissionableNodesCommissioningEnabled(chip::Controller::DeviceCommissioner * devCtrl);
+
+ChipError::StorageType pychip_DeviceController_OnNetworkCommission(chip::Controller::DeviceCommissioner * devCtrl, uint64_t nodeId,
+                                                                   uint32_t setupPasscode, const uint8_t filterType,
+                                                                   const char * filterParam);
+
 ChipError::StorageType pychip_DeviceController_PostTaskOnChipThread(ChipThreadTaskRunnerFunct callback, void * pythonContext);
 
 ChipError::StorageType pychip_DeviceController_OpenCommissioningWindow(chip::Controller::DeviceCommissioner * devCtrl,
@@ -381,6 +388,48 @@ ChipError::StorageType pychip_DeviceController_ConnectWithCode(chip::Controller:
     return devCtrl->PairDevice(nodeid, onboardingPayload, sCommissioningParameters).AsInteger();
 }
 
+ChipError::StorageType pychip_DeviceController_OnNetworkCommission(chip::Controller::DeviceCommissioner * devCtrl, uint64_t nodeId,
+                                                                   uint32_t setupPasscode, const uint8_t filterType,
+                                                                   const char * filterParam)
+{
+    Dnssd::DiscoveryFilter filter(static_cast<Dnssd::DiscoveryFilterType>(filterType));
+    switch (static_cast<Dnssd::DiscoveryFilterType>(filterType))
+    {
+    case chip::Dnssd::DiscoveryFilterType::kNone:
+        break;
+    case chip::Dnssd::DiscoveryFilterType::kShortDiscriminator:
+    case chip::Dnssd::DiscoveryFilterType::kLongDiscriminator:
+    case chip::Dnssd::DiscoveryFilterType::kCompressedFabricId:
+    case chip::Dnssd::DiscoveryFilterType::kVendorId:
+    case chip::Dnssd::DiscoveryFilterType::kDeviceType: {
+        // For any numerical filter, convert the string to a filter value
+        errno                               = 0;
+        unsigned long long int numericalArg = strtoull(filterParam, nullptr, 0);
+        if ((numericalArg == ULLONG_MAX) && (errno == ERANGE))
+        {
+            return CHIP_ERROR_INVALID_ARGUMENT.AsInteger();
+        }
+        filter.code = static_cast<uint64_t>(numericalArg);
+        break;
+    }
+    case chip::Dnssd::DiscoveryFilterType::kCommissioningMode:
+        break;
+    case chip::Dnssd::DiscoveryFilterType::kCommissioner:
+        filter.code = 1;
+        break;
+    case chip::Dnssd::DiscoveryFilterType::kInstanceName:
+        filter.code         = 0;
+        filter.instanceName = filterParam;
+        break;
+    default:
+        return CHIP_ERROR_INVALID_ARGUMENT.AsInteger();
+    }
+
+    sPairingDeviceDiscoveryDelegate.Init(nodeId, setupPasscode, sCommissioningParameters, &sPairingDelegate, devCtrl);
+    devCtrl->RegisterDeviceDiscoveryDelegate(&sPairingDeviceDiscoveryDelegate);
+    return devCtrl->DiscoverCommissionableNodes(filter).AsInteger();
+}
+
 ChipError::StorageType pychip_DeviceController_SetThreadOperationalDataset(const char * threadOperationalDataset, uint32_t size)
 {
     ReturnErrorCodeIf(!sThreadBuf.Alloc(size), CHIP_ERROR_NO_MEMORY.AsInteger());

src/controller/python/ChipDeviceController-ScriptPairingDeviceDiscoveryDelegate.cpp

@@ -0,0 +1,52 @@
+/*
+ *
+ *    Copyright (c) 2022 Project CHIP Authors
+ *    All rights reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#include "ChipDeviceController-ScriptPairingDeviceDiscoveryDelegate.h"
+
+namespace chip {
+namespace Controller {
+
+void ScriptPairingDeviceDiscoveryDelegate::OnDiscoveredDevice(const Dnssd::DiscoveredNodeData & nodeData)
+{
+    // Ignore nodes with closed comissioning window
+    VerifyOrReturn(nodeData.commissionData.commissioningMode != 0);
+    VerifyOrReturn(mActiveDeviceCommissioner != nullptr);
+
+    const uint16_t port = nodeData.resolutionData.port;
+    char buf[chip::Inet::IPAddress::kMaxStringLength];
+    nodeData.resolutionData.ipAddress[0].ToString(buf);
+    ChipLogProgress(chipTool, "Discovered Device: %s:%u", buf, port);
+
+    // Stop Mdns discovery.
+    mActiveDeviceCommissioner->RegisterDeviceDiscoveryDelegate(nullptr);
+
+    Inet::InterfaceId interfaceId =
+        nodeData.resolutionData.ipAddress[0].IsIPv6LinkLocal() ? nodeData.resolutionData.interfaceId : Inet::InterfaceId::Null();
+    PeerAddress peerAddress = PeerAddress::UDP(nodeData.resolutionData.ipAddress[0], port, interfaceId);
+
+    RendezvousParameters keyExchangeParams = RendezvousParameters().SetSetupPINCode(mSetupPasscode).SetPeerAddress(peerAddress);
+
+    CHIP_ERROR err = mActiveDeviceCommissioner->PairDevice(mNodeId, keyExchangeParams, mParams);
+    if (err != CHIP_NO_ERROR)
+    {
+        VerifyOrReturn(mPairingDelegate != nullptr);
+        mPairingDelegate->OnCommissioningComplete(mNodeId, err);
+    }
+}
+} // namespace Controller
+} // namespace chip

src/controller/python/ChipDeviceController-ScriptPairingDeviceDiscoveryDelegate.h

@@ -0,0 +1,52 @@
+/*
+ *
+ *    Copyright (c) 2022 Project CHIP Authors
+ *    All rights reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#pragma once
+
+#include "ChipDeviceController-ScriptDevicePairingDelegate.h"
+
+#include <controller/CHIPDeviceController.h>
+
+namespace chip {
+namespace Controller {
+
+class ScriptPairingDeviceDiscoveryDelegate : public DeviceDiscoveryDelegate
+{
+public:
+    void Init(NodeId nodeId, uint32_t setupPasscode, CommissioningParameters commissioningParams,
+              ScriptDevicePairingDelegate * pairingDelegate, DeviceCommissioner * activeDeviceCommissioner)
+    {
+        mNodeId                   = nodeId;
+        mSetupPasscode            = setupPasscode;
+        mParams                   = commissioningParams;
+        mPairingDelegate          = pairingDelegate;
+        mActiveDeviceCommissioner = activeDeviceCommissioner;
+    }
+    void OnDiscoveredDevice(const Dnssd::DiscoveredNodeData & nodeData);
+
+private:
+    ScriptDevicePairingDelegate * mPairingDelegate;
+    DeviceCommissioner * mActiveDeviceCommissioner = nullptr;
+
+    CommissioningParameters mParams;
+    NodeId mNodeId;
+    uint32_t mSetupPasscode;
+};
+
+} // namespace Controller
+} // namespace chip

src/controller/python/chip/ChipDeviceCtrl.py

@@ -117,6 +117,19 @@ class DCState(enum.IntEnum):
     COMMISSIONING = 5
 
 
+class DiscoveryFilterType(enum.IntEnum):
+    # These must match chip::Dnssd::DiscoveryFilterType values (barring the naming convention)
+    NONE = 0
+    SHORT_DISCRIMINATOR = 1
+    LONG_DISCRIMINATOR = 2
+    VENDOR_ID = 3
+    DEVICE_TYPE = 4
+    COMMISSIONING_MODE = 5
+    INSTANCE_NAME = 6
+    COMMISSIONER = 7
+    COMPRESSED_FABRIC_ID = 8
+
+
 class ChipDeviceController():
     activeList = set()
 
@@ -150,30 +163,36 @@ def __init__(self, opCredsContext: ctypes.c_void_p, fabricId: int, nodeId: int,
         def GetNodeId(self):
             return self.nodeId
 
+        def HandleCommissioningComplete(nodeid, err):
+            if err != 0:
+                print("Failed to commission: {}".format(err))
+            else:
+                print("Commissioning complete")
+            self.state = DCState.IDLE
+            self._ChipStack.callbackRes = err
+            self._ChipStack.commissioningEventRes = err
+            self._ChipStack.commissioningCompleteEvent.set()
+            self._ChipStack.completeEvent.set()
+
         def HandleKeyExchangeComplete(err):
             if err != 0:
                 print("Failed to establish secure session to device: {}".format(err))
                 self._ChipStack.callbackRes = self._ChipStack.ErrorToException(
                     err)
             else:
                 print("Established secure session with Device")
+
             if self.state != DCState.COMMISSIONING:
                 # During Commissioning, HandleKeyExchangeComplete will also be called,
                 # in this case the async operation should be marked as finished by
                 # HandleCommissioningComplete instead this function.
                 self.state = DCState.IDLE
                 self._ChipStack.completeEvent.set()
-
-        def HandleCommissioningComplete(nodeid, err):
-            if err != 0:
-                print("Failed to commission: {}".format(err))
             else:
-                print("Commissioning complete")
-            self.state = DCState.IDLE
-            self._ChipStack.callbackRes = err
-            self._ChipStack.commissioningEventRes = err
-            self._ChipStack.commissioningCompleteEvent.set()
-            self._ChipStack.completeEvent.set()
+                # When commissioning, getting an error during key exhange
+                # needs to unblock the entire commissioning flow.
+                if err != 0:
+                    HandleCommissioningComplete(0, err)
 
         self.cbHandleKeyExchangeCompleteFunct = _DevicePairingDelegate_OnPairingCompleteFunct(
             HandleKeyExchangeComplete)
@@ -345,6 +364,44 @@ def CheckTestCommissionerCallbacks(self):
     def CheckTestCommissionerPaseConnection(self, nodeid):
         return self._dmLib.pychip_TestPaseConnection(nodeid)
 
+    def CommissionOnNetwork(self, nodeId: int, setupPinCode: int, filterType: DiscoveryFilterType = DiscoveryFilterType.NONE, filter: typing.Any = None):
+        '''
+        Does the routine for OnNetworkCommissioning, with a filter for mDNS discovery.
+        Supported filters are:
+
+            DiscoveryFilterType.NONE
+            DiscoveryFilterType.SHORT_DISCRIMINATOR
+            DiscoveryFilterType.LONG_DISCRIMINATOR
+            DiscoveryFilterType.VENDOR_ID
+            DiscoveryFilterType.DEVICE_TYPE
+            DiscoveryFilterType.COMMISSIONING_MODE
+            DiscoveryFilterType.INSTANCE_NAME
+            DiscoveryFilterType.COMMISSIONER
+            DiscoveryFilterType.COMPRESSED_FABRIC_ID
+
+        The filter can be an integer, a string or None depending on the actual type of selected filter.
+        '''
+        self.CheckIsActive()
+
+        # IP connection will run through full commissioning, so we need to wait
+        # for the commissioning complete event, not just any callback.
+        self.state = DCState.COMMISSIONING
+
+        # Convert numerical filters to string for passing down to binding.
+        if isinstance(filter, int):
+            filter = str(filter)
+
+        self._ChipStack.commissioningCompleteEvent.clear()
+
+        self._ChipStack.CallAsync(
+            lambda: self._dmLib.pychip_DeviceController_OnNetworkCommission(
+                self.devCtrl, nodeId, setupPinCode, int(filterType), str(filter).encode("utf-8") + b"\x00" if filter is not None else None)
+        )
+        if not self._ChipStack.commissioningCompleteEvent.isSet():
+            # Error 50 is a timeout
+            return False
+        return self._ChipStack.commissioningEventRes == 0
+
     def CommissionWithCode(self, setupPayload: str, nodeid: int):
         self.CheckIsActive()
 
@@ -1076,6 +1133,9 @@ def _InitLib(self):
                 c_void_p, c_uint64]
             self._dmLib.pychip_DeviceController_Commission.restype = c_uint32
 
+            self._dmLib.pychip_DeviceController_OnNetworkCommission.argtypes = [c_void_p, c_uint64, c_uint32, c_uint8, c_char_p]
+            self._dmLib.pychip_DeviceController_OnNetworkCommission.restype = c_uint32
+
             self._dmLib.pychip_DeviceController_DiscoverAllCommissionableNodes.argtypes = [
                 c_void_p]
             self._dmLib.pychip_DeviceController_DiscoverAllCommissionableNodes.restype = c_uint32

src/controller/python/chip/ChipStack.py

@@ -324,6 +324,7 @@ def setLogFunct(self, logFunct):
     def Shutdown(self):
         # Make sure PersistentStorage is destructed before chipStack
         # to avoid accessing builtins.chipStack after destruction.
+        self._persistentStorage.Shutdown()
         self._persistentStorage = None
         self.Call(lambda: self._ChipStackLib.pychip_DeviceController_StackShutdown())
 
@@ -339,6 +340,8 @@ def Shutdown(self):
         self.devMgr = None
         self.callbackRes = None
 
+        delattr(builtins, "chipStack")
+
     def Call(self, callFunct, timeoutMs: int = None):
         '''Run a Python function on CHIP stack, and wait for the response.
         This function is a wrapper of PostTaskOnChipThread, which includes some handling of application specific logics.