From 3051345eef63b713a2822836c1b5b8e203d3ce08 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 30 Nov 2021 12:16:05 -0500 Subject: [PATCH] Better test discovery (#12292) * use chip tool to figure out exactly what tests exist rather than searching for YAML files * Fix arguments for running tests * Restyle fixes * Update to the test list command to finding out available tests * ignore empty lines * Update scripts/tests/run_test_suite.py Co-authored-by: Victor Morales * Add missing \ Co-authored-by: Justin Wood Co-authored-by: Victor Morales --- .github/workflows/tests.yaml | 7 ++++--- scripts/tests/chiptest/__init__.py | 21 +++++++++------------ scripts/tests/chiptest/test_definition.py | 1 - scripts/tests/run_test_suite.py | 22 ++++++++++++---------- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index afd197f554ce4d..e0ba2a9a6aebb1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -87,9 +87,10 @@ jobs: timeout-minutes: 30 run: | ./scripts/run_in_build_env.sh \ - "./scripts/tests/run_test_suite.py run \ - --iterations 2 \ + "./scripts/tests/run_test_suite.py \ --chip-tool ./out/debug/standalone/chip-tool \ + run \ + --iterations 2 \ --all-clusters-app ./out/debug/standalone/chip-all-clusters-app \ --tv-app ./out/debug/standalone/chip-tv-app \ " @@ -176,10 +177,10 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ + --chip-tool ./out/debug/standalone/chip-tool \ --target-skip-glob 'TV_*' \ run \ --iterations 2 \ - --chip-tool ./out/debug/standalone/chip-tool \ --all-clusters-app ./out/debug/standalone/chip-all-clusters-app \ --tv-app ./out/debug/standalone/chip-tv-app \ " diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 2d72bc2b43a4da..b21fc02f310859 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -17,6 +17,8 @@ from pathlib import Path import os import logging +import subprocess +import re import chiptest.linux import chiptest.runner @@ -24,27 +26,22 @@ from .test_definition import TestTarget, TestDefinition, ApplicationPaths -def AllTests(root: str): - """Gets all the tests that can be found in the ROOT directory based on - yaml file names. +def AllTests(chip_tool: str): + """Executes `chip_tool` binary to see what tests are available. """ - for path in Path(os.path.join(root, 'src', 'app', 'tests', 'suites')).rglob("*.yaml"): - logging.debug('Found YAML: %s' % path) - # grab the name without the extension - name = path.stem + result = subprocess.run([chip_tool, 'tests', 'list'], capture_output=True) - if 'Simulated' in name: + for name in result.stdout.decode('utf8').split('\n'): + if not name: continue if name.startswith('TV_'): target = TestTarget.TV - elif name.startswith('Test'): - target = TestTarget.ALL_CLUSTERS else: - continue + target = TestTarget.ALL_CLUSTERS - yield TestDefinition(yaml_file=path, run_name=path.stem, name=name, target=target) + yield TestDefinition(run_name=name, name=name, target=target) __all__ = ['TestTarget', 'TestDefinition', 'AllTests', 'ApplicationPaths'] diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py index 9d6e22ba6c3109..9eea86966a6e01 100644 --- a/scripts/tests/chiptest/test_definition.py +++ b/scripts/tests/chiptest/test_definition.py @@ -79,7 +79,6 @@ def LogContents(self): @dataclass class TestDefinition: - yaml_file: str name: str run_name: str target: TestTarget diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 133e9213ba7ca4..688778423f038a 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -61,6 +61,7 @@ class RunContext: root: str tests: typing.List[chiptest.TestDefinition] in_unshare: bool + chip_tool: str @click.group(chain=True) @@ -93,7 +94,7 @@ class RunContext: @click.option( '--root', default=DEFAULT_CHIP_ROOT, - help='Default directory path for CHIP. Used to determine what tests exist') + help='Default directory path for CHIP. Used to copy run configurations') @click.option( '--internal-inside-unshare', hidden=True, @@ -101,8 +102,12 @@ class RunContext: default=False, help='Internal flag for running inside a unshared environment' ) +@click.option( + '--chip-tool', + default=FindBinaryPath('chip-tool'), + help='Binary path of chip tool app to use to run the test') @click.pass_context -def main(context, log_level, target, target_glob, target_skip_glob, no_log_timestamps, root, internal_inside_unshare): +def main(context, log_level, target, target_glob, target_skip_glob, no_log_timestamps, root, internal_inside_unshare, chip_tool): # Ensures somewhat pretty logging of what is going on log_fmt = '%(asctime)s.%(msecs)03d %(levelname)-7s %(message)s' if no_log_timestamps: @@ -110,7 +115,7 @@ def main(context, log_level, target, target_glob, target_skip_glob, no_log_times coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt) # Figures out selected test that match the given name(s) - all_tests = [test for test in chiptest.AllTests(root)] + all_tests = [test for test in chiptest.AllTests(chip_tool)] tests = all_tests if 'all' not in target: tests = [] @@ -139,7 +144,8 @@ def main(context, log_level, target, target_glob, target_skip_glob, no_log_times tests.sort(key=lambda x: x.name) context.obj = RunContext(root=root, tests=tests, - in_unshare=internal_inside_unshare) + in_unshare=internal_inside_unshare, + chip_tool=chip_tool) @main.command( @@ -156,10 +162,6 @@ def cmd_generate(context): '--iterations', default=1, help='Number of iterations to run') -@click.option( - '--chip-tool', - default=FindBinaryPath('chip-tool'), - help='What chip tool app to use to run the test') @click.option( '--all-clusters-app', default=FindBinaryPath('chip-all-clusters-app'), @@ -169,12 +171,12 @@ def cmd_generate(context): default=FindBinaryPath('chip-tv-app'), help='what tv app to use') @click.pass_context -def cmd_run(context, iterations, chip_tool, all_clusters_app, tv_app): +def cmd_run(context, iterations, all_clusters_app, tv_app): runner = chiptest.runner.Runner() # Command execution requires an array paths = chiptest.ApplicationPaths( - chip_tool=[chip_tool], + chip_tool=[context.obj.chip_tool], all_clusters_app=[all_clusters_app], tv_app=[tv_app] )