From 6224301069a45de0712a87b7b70616564796f5c9 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 27 Jan 2023 23:07:55 -0500 Subject: [PATCH] [repl tests] Allow the ability to run all tests in the test suite (including manual) (#24707) * Allow the ability to run all tests in the test suite (including manual) * Even better logic for tests - allow full control on what to do with manual tests (skip, keep as is, only manual) * Update flags a bit to make it clearer what manual flag does * Typo fix * Restyle --------- Co-authored-by: Andrei Litvin --- scripts/tests/run_test_suite.py | 54 +++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 8c6df448ee7fed..3984e641fbcd9d 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import enum import logging import os import sys @@ -32,6 +33,12 @@ os.path.join(os.path.dirname(__file__), '..', '..')) +class ManualHandling(enum.Enum): + INCLUDE = enum.auto() + SKIP = enum.auto() + ONLY = enum.auto() + + def FindBinaryPath(name: str): for path in Path(DEFAULT_CHIP_ROOT).rglob(name): if not path.is_file(): @@ -60,7 +67,7 @@ class RunContext: in_unshare: bool chip_tool: str dry_run: bool - skip_manual: bool + manual_handling: ManualHandling @click.group(chain=True) @@ -106,6 +113,13 @@ class RunContext: default=False, help='Internal flag for running inside a unshared environment' ) +@click.option( + '--manual', + type=click.Choice(['skip', 'only', 'include'], case_sensitive=False), + default='skip', + show_default=True, + help='Internal flag to determine how to handle manual tests. ONLY for "all" test choice.', +) @click.option( '--run-yamltests-with-chip-repl', default=False, @@ -116,7 +130,7 @@ class RunContext: help='Binary path of chip tool app to use to run the test') @click.pass_context def main(context, dry_run, log_level, target, target_glob, target_skip_glob, - no_log_timestamps, root, internal_inside_unshare, run_yamltests_with_chip_repl, chip_tool): + no_log_timestamps, root, internal_inside_unshare, manual, run_yamltests_with_chip_repl, chip_tool): # Ensures somewhat pretty logging of what is going on log_fmt = '%(asctime)s.%(msecs)03d %(levelname)-7s %(message)s' if no_log_timestamps: @@ -133,7 +147,16 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, tests = all_tests # Default to only non-manual tests unless explicit targets are specified. - skip_manual = 'all' in target + if 'all' in target: + if manual == 'skip': + manual_handling = ManualHandling.SKIP + elif manual == 'only': + manual_handling = ManualHandling.ONLY + else: + manual_handling = ManualHandling.INCLUDE + else: + manual_handling = ManualHandling.INCLUDE + if 'all' not in target: tests = [] for name in target: @@ -145,7 +168,6 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, if target_glob: matcher = GlobMatcher(target_glob.lower()) - # Globs ignore manual tests, because it's too easy to mess up otherwise. tests = [test for test in tests if matcher.matches(test.name.lower())] if len(tests) == 0: @@ -164,7 +186,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, context.obj = RunContext(root=root, tests=tests, in_unshare=internal_inside_unshare, chip_tool=chip_tool, dry_run=dry_run, - skip_manual=skip_manual) + manual_handling=manual_handling) @main.command( @@ -209,14 +231,21 @@ def cmd_list(context): '--pics-file', type=click.Path(exists=True), default="src/app/tests/suites/certification/ci-pics-values", + show_default=True, help='PICS file to use for test runs.') +@click.option( + '--keep-going', + is_flag=True, + default=False, + show_default=True, + help='Keep running the rest of the tests even if a test fails.') @click.option( '--test-timeout-seconds', default=None, type=int, help='If provided, fail if a test runs for longer than this time') @click.pass_context -def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, ota_requestor_app, tv_app, bridge_app, chip_repl_yaml_tester, pics_file, test_timeout_seconds): +def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, ota_requestor_app, tv_app, bridge_app, chip_repl_yaml_tester, pics_file, keep_going, test_timeout_seconds): runner = chiptest.runner.Runner() if all_clusters_app is None: @@ -265,8 +294,12 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o for i in range(iterations): logging.info("Starting iteration %d" % (i+1)) for test in context.obj.tests: - if context.obj.skip_manual and test.is_manual: - continue + if test.is_manual: + if context.obj.manual_handling == ManualHandling.SKIP: + continue + else: + if context.obj.manual_handling == ManualHandling.ONLY: + continue test_start = time.monotonic() try: @@ -283,8 +316,9 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o test_end = time.monotonic() logging.exception('%-30s - FAILED in %0.2f seconds' % (test.name, (test_end - test_start))) - apps_register.uninit() - sys.exit(2) + if not keep_going: + apps_register.uninit() + sys.exit(2) apps_register.uninit() if sys.platform == 'linux':