From 510f97e753d6adc58a4ec1fbadd41523f01783ae Mon Sep 17 00:00:00 2001 From: Boris Zbarsky Date: Fri, 26 Nov 2021 02:57:11 -0500 Subject: [PATCH 1/3] Re-enable test coverage that the python test suite runner disabled. The pattern for tests is not "test_*"; there are a bunch of tests (e.g. TestCluster) that don't match that. The correct pattern is "Test*" (and "TV_*" for the TV tests). "test*" also does not work right because it includes src/app/tests/suites/certification/tests.yaml which is not a test yaml we want to be running. Summary of changes: 1. Stop lowercasing test names in creating the test definitions, so we can actually match against "Test*" sanely. This also makes the test names match what consumers see in the actual filenames. 2. Change the test name detection to match on "Test*", which fixes the actual regression. 3. Move the lowercasing to the --target and --target-glob matching. 4. Add more useful error output when unrecognized --target values are specified, and exit with a failure in that case instead of silently succeeding. --- scripts/tests/chiptest/__init__.py | 10 ++++------ scripts/tests/run_test_suite.py | 23 ++++++++++++++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 0e7223e01d4af9..2d72bc2b43a4da 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -32,17 +32,15 @@ def AllTests(root: str): logging.debug('Found YAML: %s' % path) # grab the name without the extension - name = path.stem.lower() + name = path.stem - if 'simulated' in name: + if 'Simulated' in name: continue - if name.startswith('tv_'): + if name.startswith('TV_'): target = TestTarget.TV - name = 'tv-' + name[3:] - elif name.startswith('test_'): + elif name.startswith('Test'): target = TestTarget.ALL_CLUSTERS - name = 'app-' + name[5:] else: continue diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 6022d6d50e5dcd..a521ee53d6e31d 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -110,18 +110,31 @@ def main(context, log_level, target, target_glob, target_skip_glob, no_log_times coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt) # Figures out selected test that match the given name(s) - tests = [test for test in chiptest.AllTests(root)] + all_tests = [test for test in chiptest.AllTests(root)] + tests = all_tests if 'all' not in target: - target = set([name.lower() for name in target]) - tests = [test for test in tests if test.name in target] + tests = [] + for name in target: + targeted = [test for test in all_tests if test.name.lower() + == name.lower()] + if len(targeted) == 0: + logging.error("Unknown target: %s" % name) + tests.extend(targeted) if target_glob: matcher = GlobMatcher(target_glob) - tests = [test for test in tests if matcher.matches(test.name)] + tests = [test for test in tests if matcher.matches(test.name.lower())] + + if len(tests) == 0: + logging.error("No targets match, exiting.") + logging.error("Valid targets are (case-insensitive): %s" % + (", ".join(test.name for test in all_tests))) + exit(1) if target_skip_glob: matcher = GlobMatcher(target_skip_glob) - tests = [test for test in tests if not matcher.matches(test.name)] + tests = [test for test in tests if not matcher.matches( + test.name.lower())] tests.sort(key=lambda x: x.name) From 9f55946ecf702e5412feedf7aba8d06febadd4c0 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 26 Nov 2021 08:31:16 -0500 Subject: [PATCH 2/3] Update the glob for skipping TV apps in darwin tests --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index dbabc1ff3b9763..7770cb5e70c0d0 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -174,7 +174,7 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ - --target-skip-glob 'tv-*' \ + --target-skip-glob 'TV_*' \ run \ --iterations 2 \ --chip-tool ./out/debug/standalone/chip-tool \ From 385c2eb49551b93f8d00476e6520b10eeca2e1bc Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 26 Nov 2021 09:27:55 -0500 Subject: [PATCH 3/3] make globs lowercase as well, since we match against lowercase names --- scripts/tests/run_test_suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index a521ee53d6e31d..133e9213ba7ca4 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -122,7 +122,7 @@ def main(context, log_level, target, target_glob, target_skip_glob, no_log_times tests.extend(targeted) if target_glob: - matcher = GlobMatcher(target_glob) + matcher = GlobMatcher(target_glob.lower()) tests = [test for test in tests if matcher.matches(test.name.lower())] if len(tests) == 0: @@ -132,7 +132,7 @@ def main(context, log_level, target, target_glob, target_skip_glob, no_log_times exit(1) if target_skip_glob: - matcher = GlobMatcher(target_skip_glob) + matcher = GlobMatcher(target_skip_glob.lower()) tests = [test for test in tests if not matcher.matches( test.name.lower())]