[MatterYamlTests][darwin-framework-tool] Use a list of tests to run f…

…rom scripts/tests/chiptest/__init__.py if chip_tool_python is used in conjunction with darwinframeworktool.py

.github/workflows/darwin-tests.yaml

@@ -106,6 +106,7 @@ jobs:
               run: |
                   ./scripts/run_in_build_env.sh \
                   "./scripts/tests/run_test_suite.py \
+                     --runner darwin_framework_tool_python \
                      --chip-tool ./out/darwin-x64-darwin-framework-tool-${BUILD_VARIANT_FRAMEWORK_TOOL}/darwin-framework-tool \
                      --target-skip-glob '{TestAccessControlConstraints}' \
                      run \

scripts/tests/chiptest/__init__.py

@@ -160,6 +160,56 @@ def _GetInDevelopmentTests() -> Set[str]:
     }
 
 
+def _GetDarwinFrameworkToolUnsupportedTests() -> Set[str]:
+    """Tests that fail in darwin-framework-tool for some reason"""
+    return {
+        "DL_LockUnlock",  # Events error.
+        "Test_AddNewFabricFromExistingFabric",  # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
+        "TestAttributesById",
+        "TestBasicInformation",  # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured
+        "TestClusterComplexTypes",
+        "TestClusterMultiFabric",
+        "TestCommandsById",  # Test Failure : The test expects the "UNSUPPORTED_ENDPOINT" error but the "FAILURE" error occured.
+        "TestCommissioningWindow",
+        "TestDiagnosticLogs.yaml", # darwin-framework-tool does not implement a BDXTransferServerDelegate
+        "TestDiscovery",  # darwin-framework-tool does not support dns-sd commands.
+        "TestEvents",
+        "TestEventsById",
+        "TestGroupMessaging",  # darwin-framework-tool does not support group commands.
+        "TestReadNoneSubscribeNone",  # darwin-framework-tool does not supports those commands.
+
+        "Test_TC_ACE_1_6",  # darwin-framework-tool does not support group commands.
+        "Test_TC_ACL_2_5",  # Events error.
+        "Test_TC_ACL_2_6",  # Events error.
+        "Test_TC_ACL_2_7",  # Events error.
+        "Test_TC_ACL_2_8",  # Events error.
+        "Test_TC_ACL_2_9",  # Events error.
+        "Test_TC_ACL_2_10",  # Events error.
+        "Test_TC_BINFO_2_1",  # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured
+        "Test_TC_BINFO_2_2",  # Events error.
+        "Test_TC_BRBINFO_2_1",  # Test Failure : The test expects the "UNSUPPORTED_ATTRIBUTE" error but the "FAILURE" error occured.
+        "Test_TC_DGEN_2_1",  # minValue: 1 <-- The response value (0) should be greater or equal to the constraint but 0 < 1
+        "Test_TC_DGGEN_2_3",  # Events error.
+        "Test_TC_DRLK_2_1",  # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured
+        "Test_TC_DGTHREAD_2_1",
+        "Test_TC_DGTHREAD_2_2",
+        "Test_TC_DGTHREAD_2_3",
+        "Test_TC_DGTHREAD_2_4",
+        "Test_TC_FLABEL_2_1",  # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured
+        "Test_TC_GRPKEY_2_1",  # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured
+        "Test_TC_LCFG_2_1",  # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured
+        "Test_TC_OPCREDS_3_7",  # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
+        "Test_TC_OPSTATE_2_4",  # Events error.
+        "Test_TC_SMOKECO_2_2",  # Events error.
+        "Test_TC_SMOKECO_2_3",  # Events error.
+        "Test_TC_SMOKECO_2_4",  # Events error.
+        "Test_TC_SMOKECO_2_5",  # Events error.
+        "Test_TC_SMOKECO_2_6",  # Events error.
+        "Test_TC_SC_4_1",  # darwin-framework-tool does not support dns-sd commands.
+        "Test_TC_SC_5_2",  # darwin-framework-tool does not support group commands.
+    }
+
+
 def _GetChipReplUnsupportedTests() -> Set[str]:
     """Tests that fail in chip-repl for some reason"""
     return {
@@ -284,7 +334,7 @@ def tests_with_command(chip_tool: str, is_manual: bool):
         )
 
 
-def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
+def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, dft_unsupported: bool, use_short_run_name: bool):
     """
     use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
     """
@@ -294,6 +344,7 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short
     extra_slow_tests = _GetExtraSlowTests()
     in_development_tests = _GetInDevelopmentTests()
     chip_repl_unsupported_tests = _GetChipReplUnsupportedTests()
+    dft_unsupported_tests = _GetDarwinFrameworkToolUnsupportedTests()
     purposeful_failure_tests = _GetPurposefulFailureTests()
 
     for path in _AllYamlTests():
@@ -327,6 +378,9 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short
         else:
             run_name = str(path)
 
+        if dft_unsupported and run_name in dft_unsupported_tests:
+            tags.add(TestTag.IN_DEVELOPMENT)
+
         yield TestDefinition(
             run_name=run_name,
             name=path.stem,  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
@@ -336,12 +390,17 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short
 
 
 def AllReplYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, dft_unsupported=False, use_short_run_name=False):
         yield test
 
 
 def AllChipToolYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, dft_unsupported=False, use_short_run_name=True):
+        yield test
+
+
+def AllDarwinFrameworkToolYamlTests():
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, dft_unsupported=True, use_short_run_name=True):
         yield test
 
 

scripts/tests/chiptest/test_definition.py

@@ -251,10 +251,9 @@ def to_s(self):
 
 
 class TestRunTime(Enum):
-    CHIP_TOOL_BUILTIN = auto()  # run via chip-tool built-in test commands
     CHIP_TOOL_PYTHON = auto()  # use the python yaml test parser with chip-tool
+    DARWIN_FRAMEWORK_TOOL_PYTHON = auto()  # use the python yaml test parser with chip-tool
     CHIP_REPL_PYTHON = auto()       # use the python yaml test runner
-    DARWIN_FRAMEWORK_TOOL_BUILTIN = auto()  # run via darwin-framework-tool built-in test commands
 
 
 @dataclass
@@ -281,7 +280,7 @@ def tags_str(self) -> str:
         return ", ".join([t.to_s() for t in self.tags])
 
     def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
-            timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_BUILTIN):
+            timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_PYTHON):
         """
         Executes the given test case using the provided runner for execution.
         """
@@ -333,8 +332,6 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
                     # so it will be commissionable again.
                     app.factoryReset()
 
-            tool_cmd = paths.chip_tool if test_runtime != TestRunTime.CHIP_TOOL_PYTHON else paths.chip_tool_with_python_cmd
-
             if dry_run:
                 tool_storage_dir = None
                 tool_storage_args = []
@@ -350,38 +347,37 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
                 app.start()
                 setupCode = app.setupCode
 
-            pairing_cmd = tool_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
-            test_cmd = tool_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
-            if test_runtime == TestRunTime.CHIP_TOOL_PYTHON:
+            if test_runtime == TestRunTime.CHIP_REPL_PYTHON:
+                chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
+                python_cmd = chip_repl_yaml_tester_cmd + \
+                    ['--setup-code', setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
+                if dry_run:
+                    logging.info(" ".join(python_cmd))
+                else:
+                    runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
+                                         dependencies=[apps_register], timeout_seconds=timeout_seconds)
+            else:
+                pairing_cmd = paths.chip_tool_with_python_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
+                test_cmd = paths.chip_tool_with_python_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
                 server_args = ['--server_path', paths.chip_tool[-1]] + \
                     ['--server_arguments', 'interactive server' +
                         (' ' if len(tool_storage_args) else '') + ' '.join(tool_storage_args)]
                 pairing_cmd += server_args
                 test_cmd += server_args
-            elif test_runtime == TestRunTime.CHIP_TOOL_BUILTIN:
-                pairing_cmd += tool_storage_args
-                test_cmd += tool_storage_args
-
-            if dry_run:
-                # Some of our command arguments have spaces in them, so if we are
-                # trying to log commands people can run we should quote those.
-                def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
-                logging.info(" ".join(map(quoter, pairing_cmd)))
-                logging.info(" ".join(map(quoter, test_cmd)))
-            elif test_runtime == TestRunTime.CHIP_REPL_PYTHON:
-                chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
-                python_cmd = chip_repl_yaml_tester_cmd + \
-                    ['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
-                runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
-                                     dependencies=[apps_register], timeout_seconds=timeout_seconds)
-            else:
-                runner.RunSubprocess(pairing_cmd,
-                                     name='PAIR', dependencies=[apps_register])
 
-                runner.RunSubprocess(
-                    test_cmd,
-                    name='TEST', dependencies=[apps_register],
-                    timeout_seconds=timeout_seconds)
+                if dry_run:
+                    # Some of our command arguments have spaces in them, so if we are
+                    # trying to log commands people can run we should quote those.
+                    def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
+                    logging.info(" ".join(map(quoter, pairing_cmd)))
+                    logging.info(" ".join(map(quoter, test_cmd)))
+                else:
+                    runner.RunSubprocess(pairing_cmd,
+                                         name='PAIR', dependencies=[apps_register])
+                    runner.RunSubprocess(
+                        test_cmd,
+                        name='TEST', dependencies=[apps_register],
+                        timeout_seconds=timeout_seconds)
 
         except Exception:
             logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!")

scripts/tests/run_test_suite.py

@@ -123,8 +123,8 @@ class RunContext:
 )
 @click.option(
     '--runner',
-    type=click.Choice(['codegen', 'chip_repl_python', 'chip_tool_python'], case_sensitive=False),
-    default='codegen',
+    type=click.Choice(['chip_repl_python', 'chip_tool_python', 'darwin_framework_tool_python'], case_sensitive=False),
+    default='chip_tool_python',
     help='Run YAML tests using the specified runner.')
 @click.option(
     '--chip-tool',
@@ -138,18 +138,18 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
         log_fmt = '%(levelname)-7s %(message)s'
     coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)
 
-    runtime = TestRunTime.CHIP_TOOL_BUILTIN
+    runtime = TestRunTime.CHIP_TOOL_PYTHON
     if runner == 'chip_repl_python':
         runtime = TestRunTime.CHIP_REPL_PYTHON
-    elif runner == 'chip_tool_python':
-        runtime = TestRunTime.CHIP_TOOL_PYTHON
-    elif chip_tool is not None and os.path.basename(chip_tool) == "darwin-framework-tool":
-        runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_BUILTIN
+    elif runner == 'darwin_framework_tool_python':
+        runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON
 
     if chip_tool is None and not runtime == TestRunTime.CHIP_REPL_PYTHON:
-        # non yaml tests REQUIRE chip-tool. Yaml tests should not require chip-tool
         paths_finder = PathsFinder()
-        chip_tool = paths_finder.get('chip-tool')
+        if runtime == TestRunTime.CHIP_TOOL_PYTHON:
+            chip_tool = paths_finder.get('chip-tool')
+        else:
+            chip_tool = paths_finder.get('darwin-framework-tool')
 
     if include_tags:
         include_tags = set([TestTag.__members__[t] for t in include_tags])
@@ -160,10 +160,10 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
     # Figures out selected test that match the given name(s)
     if runtime == TestRunTime.CHIP_REPL_PYTHON:
         all_tests = [test for test in chiptest.AllReplYamlTests()]
-    elif runtime == TestRunTime.CHIP_TOOL_PYTHON and os.path.basename(chip_tool) != "darwin-framework-tool":
-        all_tests = [test for test in chiptest.AllChipToolYamlTests()]
+    elif runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
+        all_tests = [test for test in chiptest.AllDarwinFrameworkToolYamlTests()]
     else:
-        all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)]
+        all_tests = [test for test in chiptest.AllChipToolYamlTests()]
 
     tests = all_tests
 
@@ -178,7 +178,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
             TestTag.PURPOSEFUL_FAILURE,
         }
 
-        if runtime != TestRunTime.CHIP_TOOL_PYTHON:
+        if runtime == TestRunTime.CHIP_REPL_PYTHON:
             exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)
 
     if 'all' not in target:
@@ -319,7 +319,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o
         chip_repl_yaml_tester = paths_finder.get('yamltest_with_chip_repl_tester.py')
 
     if chip_tool_with_python is None:
-        if context.obj.chip_tool and os.path.basename(context.obj.chip_tool) == "darwin-framework-tool":
+        if context.obj.runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
             chip_tool_with_python = paths_finder.get('darwinframeworktool.py')
         else:
             chip_tool_with_python = paths_finder.get('chiptool.py')