[YAML] Expose methods to start/stop/reboot the accessory used for tes…

…ting (#14240)

* [Test Suites] Expose methods to start/stop/reboot the accessory used for testing

* Add SystemCommands interface to src/app/tests/suites/commands/system

* [chip-tool] Add TestSystemCommands.yaml

* Update generated test code

examples/chip-tool/BUILD.gn

@@ -70,6 +70,7 @@ static_library("chip-tool-utils") {
   public_deps = [
     "${chip_root}/src/app/server",
     "${chip_root}/src/app/tests/suites/commands/log",
+    "${chip_root}/src/app/tests/suites/commands/system",
     "${chip_root}/src/app/tests/suites/pics",
     "${chip_root}/src/controller/data_model",
     "${chip_root}/src/lib",

examples/chip-tool/commands/tests/TestCommand.h

@@ -20,6 +20,7 @@
 
 #include "../common/CHIPCommand.h"
 #include <app/tests/suites/commands/log/LogCommands.h>
+#include <app/tests/suites/commands/system/SystemCommands.h>
 #include <app/tests/suites/include/ConstraintsChecker.h>
 #include <app/tests/suites/include/PICSChecker.h>
 #include <app/tests/suites/include/ValueChecker.h>
@@ -28,7 +29,12 @@
 
 constexpr uint16_t kTimeoutInSeconds = 90;
 
-class TestCommand : public CHIPCommand, public ValueChecker, public ConstraintsChecker, public PICSChecker, public LogCommands
+class TestCommand : public CHIPCommand,
+                    public ValueChecker,
+                    public ConstraintsChecker,
+                    public PICSChecker,
+                    public LogCommands,
+                    public SystemCommands
 {
 public:
     TestCommand(const char * commandName, CredentialIssuerCommands * credsIssuerConfig) :

examples/chip-tool/templates/tests.js

@@ -234,6 +234,7 @@ function getTests()
     'TestIdentifyCluster',
     'TestOperationalCredentialsCluster',
     'TestModeSelectCluster',
+    'TestSystemCommands',
   ];
 
   const SoftwareDiagnostics = [

scripts/tests/chiptest/accessories.py

@@ -0,0 +1,108 @@
+#
+#    Copyright (c) 2021 Project CHIP Authors
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import logging
+import time
+import threading
+import sys
+from random import randrange
+from xmlrpc.server import SimpleXMLRPCServer
+from xmlrpc.client import ServerProxy
+
+IP = '127.0.0.1'
+PORT = 9000
+
+if sys.platform == 'linux':
+    IP = '10.10.10.5'
+
+
+class AppsRegister:
+    _instance = None
+    __accessories = {}
+
+    def init(self):
+        self.__startXMLRPCServer()
+
+    def uninit(self):
+        self.__stopXMLRPCServer()
+
+    def add(self, name, accessory):
+        self.__accessories[name] = accessory
+
+    def remove(self, name):
+        self.__accessories.pop(name)
+
+    def removeAll(self):
+        self.__accessories = {}
+
+    def poll(self):
+        for accessory in self.__accessories.values():
+            status = accessory.poll()
+            if status is not None:
+                return status
+        return None
+
+    def kill(self, name):
+        accessory = self.__accessories[name]
+        if accessory:
+            accessory.kill()
+
+    def killAll(self):
+        for accessory in self.__accessories.values():
+            accessory.kill()
+
+    def start(self, name, discriminator):
+        accessory = self.__accessories[name]
+        if accessory:
+            return accessory.start(discriminator)
+        return False
+
+    def stop(self, name):
+        accessory = self.__accessories[name]
+        if accessory:
+            return accessory.stop()
+        return False
+
+    def reboot(self, name, discriminator):
+        accessory = self.__accessories[name]
+        if accessory:
+            return accessory.stop() and accessory.start(discriminator)
+        return False
+
+    def ping(self):
+        return True
+
+    def __startXMLRPCServer(self):
+        self.server = SimpleXMLRPCServer((IP, PORT))
+
+        self.server.register_function(self.start, 'start')
+        self.server.register_function(self.stop, 'stop')
+        self.server.register_function(self.reboot, 'reboot')
+        self.server.register_function(self.ping, 'ping')
+
+        self.server_thread = threading.Thread(target=self.__handle_request)
+        self.server_thread.start()
+
+    def __handle_request(self):
+        self.__should_handle_requests = True
+        while self.__should_handle_requests:
+            self.server.handle_request()
+
+    def __stopXMLRPCServer(self):
+        self.__should_handle_requests = False
+        # handle_request will wait until it receives a message, so let's send a ping to the server
+        client = ServerProxy('http://' + IP + ':' +
+                             str(PORT) + '/', allow_none=True)
+        client.ping()

scripts/tests/chiptest/linux.py

@@ -68,6 +68,7 @@ def CreateNamespacesForAppTest():
         # create links for switch to net connections
         "ip link add eth-app type veth peer name eth-app-switch",
         "ip link add eth-tool type veth peer name eth-tool-switch",
+        "ip link add eth-ci type veth peer name eth-ci-switch",
 
         # link the connections together
         "ip link set eth-app netns app",
@@ -77,6 +78,7 @@ def CreateNamespacesForAppTest():
         "ip link set br1 up",
         "ip link set eth-app-switch master br1",
         "ip link set eth-tool-switch master br1",
+        "ip link set eth-ci-switch master br1",
 
         # mark connections up
         "ip netns exec app ip addr add 10.10.10.1/24 dev eth-app",
@@ -94,6 +96,11 @@ def CreateNamespacesForAppTest():
         "ip netns exec app ip -6 addr flush eth-app",
         "ip netns exec tool ip -6 a add fd00:0:1:1::2/64 dev eth-tool",
         "ip netns exec app ip -6 a add fd00:0:1:1::3/64 dev eth-app",
+
+        # create link between virtual host 'tool' and the test runner
+        "ip addr add 10.10.10.5/24 dev eth-ci",
+        "ip link set dev eth-ci up",
+        "ip link set dev eth-ci-switch up",
     ]
 
     for command in COMMANDS:

scripts/tests/chiptest/test_definition.py

@@ -29,6 +29,81 @@
 TEST_NODE_ID = '0x12344321'
 
 
+class App:
+    def __init__(self, runner, command):
+        self.process = None
+        self.runner = runner
+        self.command = command
+        self.stopped = False
+
+    def start(self, discriminator):
+        if not self.process:
+            self.process = None
+            process, outpipe, errpipe = self.__startServer(
+                self.runner, self.command, discriminator)
+            self.__waitForServerReady(outpipe)
+            self.__updateSetUpCode(outpipe)
+            self.process = process
+            self.stopped = False
+            return True
+        return False
+
+    def stop(self):
+        if self.process:
+            self.stopped = True
+            self.process.kill()
+            self.process.wait(10)
+            self.process = None
+            return True
+        return False
+
+    def reboot(self, discriminator):
+        if self.process:
+            self.stop()
+            self.start(discriminator)
+            return True
+        return False
+
+    def poll(self):
+        # When the server is manually stopped, process polling is overriden so the other
+        # processes that depends on the accessory beeing alive does not stop.
+        if self.stopped:
+            return None
+        return self.process.poll()
+
+    def kill(self):
+        if self.process:
+            self.process.kill()
+
+    def wait(self, duration):
+        if self.process:
+            self.process.wait(duration)
+
+    def __startServer(self, runner, command, discriminator):
+        logging.debug(
+            'Executing application under test with discriminator %s.' % discriminator)
+        app_cmd = command + ['--discriminator', str(discriminator)]
+        return runner.RunSubprocess(app_cmd, name='APP ', wait=False)
+
+    def __waitForServerReady(self, outpipe):
+        logging.debug('Waiting for server to listen.')
+        start_time = time.time()
+        server_is_listening = outpipe.CapturedLogContains("Server Listening")
+        while not server_is_listening:
+            if time.time() - start_time > 10:
+                raise Exception('Timeout for server listening')
+            time.sleep(0.1)
+            server_is_listening = outpipe.CapturedLogContains(
+                "Server Listening")
+        logging.debug('Server is listening. Can proceed.')
+
+    def __updateSetUpCode(self, outpipe):
+        qrLine = outpipe.FindLastMatchingLine('.*SetupQRCode: *\\[(.*)]')
+        if not qrLine:
+            raise Exception("Unable to find QR code")
+        self.setupCode = qrLine.group(1)
+
+
 class TestTarget(Enum):
     ALL_CLUSTERS = auto()
     TV = auto()
@@ -87,9 +162,8 @@ class TestDefinition:
     run_name: str
     target: TestTarget
 
-    def Run(self, runner, paths: ApplicationPaths):
+    def Run(self, runner, apps_register, paths: ApplicationPaths):
         """Executes the given test case using the provided runner for execution."""
-        app_process = None
         runner.capture_delegate = ExecutionCapture()
 
         try:
@@ -127,37 +201,19 @@ def Run(self, runner, paths: ApplicationPaths):
                 if os.path.exists(str(Path.home()) + '/Documents/chip.store'):
                     os.unlink(str(Path.home()) + '/Documents/chip.store')
 
-            discriminator = str(randrange(1, 4096))
-            logging.debug(
-                'Executing application under test with discriminator %s.' % discriminator)
-            app_process, outpipe, errpipe = runner.RunSubprocess(
-                app_cmd + ['--discriminator', discriminator], name='APP ', wait=False)
+            app = App(runner, app_cmd)
+            app.start(str(randrange(1, 4096)))
+            apps_register.add("default", app)
 
-            logging.debug('Waiting for server to listen.')
-            start_time = time.time()
-            server_is_listening = outpipe.CapturedLogContains(
-                "Server Listening")
-            while not server_is_listening:
-                if time.time() - start_time > 10:
-                    raise Exception('Timeout for server listening')
-                time.sleep(0.1)
-                server_is_listening = outpipe.CapturedLogContains(
-                    "Server Listening")
-            logging.debug('Server is listening. Can proceed.')
-            qrLine = outpipe.FindLastMatchingLine('.*SetupQRCode: *\\[(.*)]')
-            if not qrLine:
-                raise Exception("Unable to find QR code")
-
-            runner.RunSubprocess(tool_cmd + ['pairing', 'qrcode', TEST_NODE_ID, qrLine.group(1)],
-                                 name='PAIR', dependencies=[app_process])
+            runner.RunSubprocess(tool_cmd + ['pairing', 'qrcode', TEST_NODE_ID, app.setupCode],
+                                 name='PAIR', dependencies=[apps_register])
 
             runner.RunSubprocess(tool_cmd + ['tests', self.run_name, TEST_NODE_ID],
-                                 name='TEST', dependencies=[app_process])
+                                 name='TEST', dependencies=[apps_register])
         except:
             logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!")
             runner.capture_delegate.LogContents()
             raise
         finally:
-            if app_process:
-                app_process.kill()
-                app_process.wait(10)
+            apps_register.killAll()
+            apps_register.removeAll()

scripts/tests/run_test_suite.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from chiptest.accessories import AppsRegister
 import coloredlogs
 import click
 import logging
@@ -195,12 +196,15 @@ def cmd_run(context, iterations, all_clusters_app, tv_app):
 
     logging.info("Each test will be executed %d times" % iterations)
 
+    apps_register = AppsRegister()
+    apps_register.init()
+
     for i in range(iterations):
         logging.info("Starting iteration %d" % (i+1))
         for test in context.obj.tests:
             test_start = time.time()
             try:
-                test.Run(runner, paths)
+                test.Run(runner, apps_register, paths)
                 test_end = time.time()
                 logging.info('%-20s - Completed in %0.2f seconds' %
                              (test.name, (test_end - test_start)))
@@ -210,6 +214,8 @@ def cmd_run(context, iterations, all_clusters_app, tv_app):
                                   (test.name, (test_end - test_start)))
                 sys.exit(2)
 
+    apps_register.uninit()
+
 
 # On linux, allow an execution shell to be prepared
 if sys.platform == 'linux':

src/app/tests/suites/TestSystemCommands.yaml

@@ -0,0 +1,41 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: System Commands Tests
+
+config:
+    cluster: "SystemCommands"
+    endpoint: 0
+
+tests:
+    - label: "Wait for the commissioned device to be retrieved"
+      cluster: "DelayCommands"
+      command: "WaitForCommissionee"
+
+    - label: "Stop the accessory"
+      command: "Stop"
+
+    - label: "Start the accessory with a given discriminator"
+      command: "Start"
+      arguments:
+          values:
+              - name: "discriminator"
+                value: 1111
+
+    - label: "Reboot the accessory with an other given discriminator"
+      command: "Reboot"
+      arguments:
+          values:
+              - name: "discriminator"
+                value: 2222