Remove wait time, update doc

.github/workflows/tests.yaml

@@ -270,7 +270,7 @@ jobs:
             - name: Run Tests
               timeout-minutes: 30
               run: |
-                    scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset -- -t 90 --disable-test ClusterObjectTests.TestTimedRequestTimeout'
+                    scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset -- -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout'
             - name: Uploading core files
               uses: actions/upload-artifact@v2
               if: ${{ failure() }} && ${{ !env.ACT }}
@@ -352,7 +352,7 @@ jobs:
             - name: Run Tests
               timeout-minutes: 30
               run: |
-                    scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset --app-params "--discriminator 3840 --interface-id -1" -- -t 90 --disable-test ClusterObjectTests.TestTimedRequestTimeout'
+                    scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset --app-params "--discriminator 3840 --interface-id -1" -- -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout'
             - name: Uploading core files
               uses: actions/upload-artifact@v2
               if: ${{ failure() }} && ${{ !env.ACT }}

docs/guides/matter-repl.md

@@ -176,14 +176,30 @@ mobile-device-test.py provides the following options for running the tests:
 ```
   --controller-nodeid INTEGER     NodeId of the controller.
   --device-nodeid INTEGER         NodeId of the device.
-  -t, --timeout INTEGER           The program will return with timeout after specified seconds.
+  -a, --address TEXT              Skip commissionee discovery, commission the
+                                  device with the IP directly.
+
+  -t, --timeout INTEGER           The program will return with timeout after
+                                  specified seconds.
+
   --discriminator INTEGER         Discriminator of the device.
   --setup-pin INTEGER             Setup pincode of the device.
-  --enable-test TEXT              The tests to be executed.
-  --disable-test TEXT             The tests to be excluded.
+  --enable-test TEXT              The tests to be executed. By default, all
+                                  tests will be executed, use this option to
+                                  run a specific set of tests. Use --print-
+                                  test-list for a list of appliable tests.
+
+  --disable-test TEXT             The tests to be excluded from the set of
+                                  enabled tests. Use --print-test-list for a
+                                  list of appliable tests.
+
   --log-level [ERROR|WARN|INFO|DEBUG]
                                   The log level of the test.
   --log-format TEXT               Override logging format
+  --print-test-list               Print a list of test cases and test sets
+                                  that can be toggled via --enable-test and
+                                  --disable-test, then exit
+
   --help                          Show this message and exit.
 ```
 
@@ -208,13 +224,16 @@ example, you can run:
 It provides some extra options, for example:
 
 ```
-  --app TEXT                  Local application to use, omit to use external
-                              apps.
-  --factoryreset              Remove /tmp/chip* before running the tests.
-  --wait-before-test INTEGER  Time for the device to start before running the
-                              tests.
-  --script PATH               Test script to use.
-  --help                      Show this message and exit.
+  --app TEXT         Local application to use, omit to use external apps, use
+                     a path for a specific binary or use a filename to search
+                     under the current matter checkout.
+
+  --factoryreset     Remove app config and repl configs (/tmp/chip* and
+                     /tmp/repl*) before running the tests.
+
+  --app-params TEXT  The extra parameters passed to the device.
+  --script PATH      Test script to use.
+  --help             Show this message and exit.
 ```
 
 You can pass your own flags for mobile-device-test.py by appending them to the

scripts/tests/run_python_test.py

@@ -64,26 +64,31 @@ def RedirectQueueThread(fp, tag, queue) -> threading.Thread:
     return log_queue_thread
 
 
-def DumpLogOutput(q: queue.Queue, timeout: int = 0):
-    deadline = time.time() + timeout
+def DumpLogOutput(q: queue.Queue):
+    # TODO: Due to the nature of os pipes, the order of the timestamp is not guaranteed, need to figure out a better output format.
     while True:
-        line = q.get((time.time() - deadline) > 0,
-                     max(deadline - time.time(), 0))
+        line = q.get_nowait()
         sys.stdout.buffer.write(
             (f"[{line[2]}]").encode() + line[0] + line[1])
         sys.stdout.flush()
 
 
+def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: str, process: subprocess.Popen, queue: queue.Queue):
+    thread_list.append(RedirectQueueThread(process.stdout,
+                                           (f"[{tag}][\33[33mSTDOUT\33[0m]").encode(), queue))
+    thread_list.append(RedirectQueueThread(process.stderr,
+                                           (f"[{tag}][\33[31mSTDERR\33[0m]").encode(), queue))
+
+
 @click.command()
-@click.option("--app", type=str, default=None, help='Local application to use, omit to use external apps.')
-@click.option("--factoryreset", is_flag=True, help='Remove /tmp/chip* before running the tests.')
-@click.option("--wait-before-test", type=int, default=3, help='Time for the device to start before running the tests.')
+@click.option("--app", type=str, default=None, help='Local application to use, omit to use external apps, use a path for a specific binary or use a filename to search under the current matter checkout.')
+@click.option("--factoryreset", is_flag=True, help='Remove app config and repl configs (/tmp/chip* and /tmp/repl*) before running the tests.')
 @click.option("--app-params", type=str, default='', help='The extra parameters passed to the device.')
 @click.option("--script", type=click.Path(exists=True), default=FindBinaryPath("mobile-device-test.py"), help='Test script to use.')
 @click.argument("script-args", nargs=-1, type=str)
-def main(app: str, factoryreset: bool, wait_before_test: int, app_params: str, script: str, script_args: typing.List[str]):
+def main(app: str, factoryreset: bool, app_params: str, script: str, script_args: typing.List[str]):
     if factoryreset:
-        retcode = subprocess.call("rm -rf /tmp/chip*", shell=True)
+        retcode = subprocess.call("rm -rf /tmp/chip* /tmp/repl*", shell=True)
         if retcode != 0:
             raise Exception("Failed to remove /tmp/chip* for factory reset.")
 
@@ -92,32 +97,24 @@ def main(app: str, factoryreset: bool, wait_before_test: int, app_params: str, s
 
     app_process = None
     if app:
-        app = FindBinaryPath(app)
-        if app is None:
-            raise FileNotFoundError(f"{app} not found")
+        if not os.path.exists(app):
+            app = FindBinaryPath(app)
+            if app is None:
+                raise FileNotFoundError(f"{app} not found")
         app_args = [app] + shlex.split(app_params)
         logging.info(f"Execute: {app_args}")
         app_process = subprocess.Popen(
             app_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)
-        log_cooking_threads.append(RedirectQueueThread(app_process.stdout,
-                                                       b"[\33[34mAPP \33[0m][\33[33mSTDOUT\33[0m]", log_queue))
-        log_cooking_threads.append(RedirectQueueThread(app_process.stderr,
-                                                       b"[\33[34mAPP \33[0m][\33[31mSTDERR\33[0m]", log_queue))
-
-    try:
-        DumpLogOutput(log_queue, wait_before_test)
-    except queue.Empty:
-        pass
+        DumpProgramOutputToQueue(
+            log_cooking_threads, "\33[34mAPP \33[0m", test_script_process, log_queue)
 
     script_command = ["/usr/bin/env", "python3", script,
                       '--log-format', '%(message)s'] + [v for v in script_args]
     logging.info(f"Execute: {script_command}")
     test_script_process = subprocess.Popen(
         script_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    log_cooking_threads.append(RedirectQueueThread(test_script_process.stdout,
-                                                   b"[\33[32mTEST\33[0m][\33[33mSTDOUT\33[0m]", log_queue))
-    log_cooking_threads.append(RedirectQueueThread(test_script_process.stderr,
-                                                   b"[\33[32mTEST\33[0m][\33[31mSTDERR\33[0m]", log_queue))
+    DumpProgramOutputToQueue(log_cooking_threads, "\33[32mTEST\33[0m",
+                             test_script_process, log_queue)
 
     test_script_exit_code = test_script_process.poll()
     while test_script_exit_code is None:

src/controller/python/test/test_scripts/mobile-device-test.py

@@ -193,8 +193,8 @@ def do_tests(controller_nodeid, device_nodeid, address, timeout, discriminator,
 @click.option("--timeout", "-t", default=240, type=int, help="The program will return with timeout after specified seconds.")
 @click.option("--discriminator", default=TEST_DISCRIMINATOR, type=int, help="Discriminator of the device.")
 @click.option("--setup-pin", default=TEST_SETUPPIN, type=int, help="Setup pincode of the device.")
-@click.option('--enable-test', default=['all'], type=click.Choice(['all'] + base.configurable_tests() + base.configurable_test_cases()), multiple=True, help='The tests to be executed. By default, all tests will be executed, use this option to run a specific set of tests.')
-@click.option('--disable-test', default=[], type=click.Choice(base.configurable_tests() + base.configurable_test_cases()), multiple=True, help='The tests to be excluded from the set of enabled tests.')
+@click.option('--enable-test', default=['all'], type=str, multiple=True, help='The tests to be executed. By default, all tests will be executed, use this option to run a specific set of tests. Use --print-test-list for a list of appliable tests.')
+@click.option('--disable-test', default=[], type=str, multiple=True, help='The tests to be excluded from the set of enabled tests. Use --print-test-list for a list of appliable tests.')
 @click.option('--log-level', default='WARN', type=click.Choice(['ERROR', 'WARN', 'INFO', 'DEBUG']), help="The log level of the test.")
 @click.option('--log-format', default=None, type=str, help="Override logging format")
 @click.option('--print-test-list', is_flag=True, help="Print a list of test cases and test sets that can be toggled via --enable-test and --disable-test, then exit")