Re-enable test coverage that the python test suite runner disabled. (#…

…12282)

* Re-enable test coverage that the python test suite runner disabled.

The pattern for tests is not "test_*"; there are a bunch of tests
(e.g. TestCluster) that don't match that.  The correct pattern is
"Test*" (and "TV_*" for the TV tests).  "test*" also does not work
right because it includes
src/app/tests/suites/certification/tests.yaml which is not a test yaml
we want to be running.

Summary of changes:

1. Stop lowercasing test names in creating the test definitions, so we
   can actually match against "Test*" sanely.  This also makes the test
   names match what consumers see in the actual filenames.
2. Change the test name detection to match on "Test*", which fixes the actual
   regression.
3. Move the lowercasing to the --target and --target-glob matching.
4. Add more useful error output when unrecognized --target values are
   specified, and exit with a failure in that case instead of silently
   succeeding.

* Update the glob for skipping TV apps in darwin tests

* make globs lowercase as well, since we match against lowercase names

Co-authored-by: Andrei Litvin <[email protected]>

.github/workflows/tests.yaml

@@ -174,7 +174,7 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                   "./scripts/tests/run_test_suite.py \
-                     --target-skip-glob 'tv-*' \
+                     --target-skip-glob 'TV_*' \
                      run \
                      --iterations 2 \
                      --chip-tool ./out/debug/standalone/chip-tool \

scripts/tests/chiptest/__init__.py

@@ -32,17 +32,15 @@ def AllTests(root: str):
         logging.debug('Found YAML: %s' % path)
 
         # grab the name without the extension
-        name = path.stem.lower()
+        name = path.stem
 
-        if 'simulated' in name:
+        if 'Simulated' in name:
             continue
 
-        if name.startswith('tv_'):
+        if name.startswith('TV_'):
             target = TestTarget.TV
-            name = 'tv-' + name[3:]
-        elif name.startswith('test_'):
+        elif name.startswith('Test'):
             target = TestTarget.ALL_CLUSTERS
-            name = 'app-' + name[5:]
         else:
             continue
 

scripts/tests/run_test_suite.py

@@ -110,18 +110,31 @@ def main(context, log_level, target, target_glob, target_skip_glob, no_log_times
     coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)
 
     # Figures out selected test that match the given name(s)
-    tests = [test for test in chiptest.AllTests(root)]
+    all_tests = [test for test in chiptest.AllTests(root)]
+    tests = all_tests
     if 'all' not in target:
-        target = set([name.lower() for name in target])
-        tests = [test for test in tests if test.name in target]
+        tests = []
+        for name in target:
+            targeted = [test for test in all_tests if test.name.lower()
+                        == name.lower()]
+            if len(targeted) == 0:
+                logging.error("Unknown target: %s" % name)
+            tests.extend(targeted)
 
     if target_glob:
-        matcher = GlobMatcher(target_glob)
-        tests = [test for test in tests if matcher.matches(test.name)]
+        matcher = GlobMatcher(target_glob.lower())
+        tests = [test for test in tests if matcher.matches(test.name.lower())]
+
+    if len(tests) == 0:
+        logging.error("No targets match, exiting.")
+        logging.error("Valid targets are (case-insensitive): %s" %
+                      (", ".join(test.name for test in all_tests)))
+        exit(1)
 
     if target_skip_glob:
-        matcher = GlobMatcher(target_skip_glob)
-        tests = [test for test in tests if not matcher.matches(test.name)]
+        matcher = GlobMatcher(target_skip_glob.lower())
+        tests = [test for test in tests if not matcher.matches(
+            test.name.lower())]
 
     tests.sort(key=lambda x: x.name)