diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 3180be2da54847..2065557c01f2f4 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -163,6 +163,54 @@ def _GetInDevelopmentTests() -> Set[str]: } +def _GetDarwinFrameworkToolUnsupportedTests() -> Set[str]: + """Tests that fail in darwin-framework-tool for some reason.""" + return { + "DL_LockUnlock", # Events error. + "Test_AddNewFabricFromExistingFabric", # darwin-framework-tool does not support the GetCommissionerRootCertificate command. + "TestAttributesById", + "TestClusterComplexTypes", + "TestClusterMultiFabric", + "TestCommandsById", # Test Failure : The test expects the "UNSUPPORTED_ENDPOINT" error but the "FAILURE" error occured. + "TestCommissioningWindow", + "TestDiscovery", # darwin-framework-tool does not support dns-sd commands. + "TestEvents", + "TestEventsById", + "TestGroupMessaging", # darwin-framework-tool does not support group commands. + "TestReadNoneSubscribeNone", # darwin-framework-tool does not supports those commands. + + "Test_TC_ACE_1_6", # darwin-framework-tool does not support group commands. + "Test_TC_ACL_2_5", # Events error. + "Test_TC_ACL_2_6", # Events error. + "Test_TC_ACL_2_7", # Events error. + "Test_TC_ACL_2_8", # Events error. + "Test_TC_ACL_2_9", # Events error. + "Test_TC_ACL_2_10", # Events error. + "Test_TC_BINFO_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_BINFO_2_2", # Events error. + "Test_TC_BRBINFO_2_1", # Test Failure : The test expects the "UNSUPPORTED_ATTRIBUTE" error but the "FAILURE" error occured. + "Test_TC_DGEN_2_1", # minValue: 1 <-- The response value (0) should be greater or equal to the constraint but 0 < 1 + "Test_TC_DGGEN_2_3", # Events error. + "Test_TC_DRLK_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_DGTHREAD_2_1", + "Test_TC_DGTHREAD_2_2", + "Test_TC_DGTHREAD_2_3", + "Test_TC_DGTHREAD_2_4", + "Test_TC_FLABEL_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_GRPKEY_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_LCFG_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_OPCREDS_3_7", # darwin-framework-tool does not support the GetCommissionerRootCertificate command. + "Test_TC_OPSTATE_2_4", # Events error. + "Test_TC_SMOKECO_2_2", # Events error. + "Test_TC_SMOKECO_2_3", # Events error. + "Test_TC_SMOKECO_2_4", # Events error. + "Test_TC_SMOKECO_2_5", # Events error. + "Test_TC_SMOKECO_2_6", # Events error. + "Test_TC_SC_4_1", # darwin-framework-tool does not support dns-sd commands. + "Test_TC_SC_5_2", # darwin-framework-tool does not support group commands. + } + + def _GetChipReplUnsupportedTests() -> Set[str]: """Tests that fail in chip-repl for some reason""" return { @@ -271,7 +319,7 @@ def tests_with_command(chip_tool: str, is_manual: bool): ) -def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool): +def _AllFoundYamlTests(use_short_run_name: bool): """ use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml" """ @@ -280,7 +328,6 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short slow_tests = _GetSlowTests() extra_slow_tests = _GetExtraSlowTests() in_development_tests = _GetInDevelopmentTests() - chip_repl_unsupported_tests = _GetChipReplUnsupportedTests() purposeful_failure_tests = _GetPurposefulFailureTests() for path in _AllYamlTests(): @@ -306,9 +353,6 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short if path.name in purposeful_failure_tests: tags.add(TestTag.PURPOSEFUL_FAILURE) - if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests: - tags.add(TestTag.IN_DEVELOPMENT) - if use_short_run_name: run_name = path.stem # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" else: @@ -323,12 +367,26 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short def AllReplYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False): + unsupported_tests = _GetChipReplUnsupportedTests() + for test in _AllFoundYamlTests(use_short_run_name=False): + + if Path(test.run_name).name in unsupported_tests: + test.tags.add(TestTag.IN_DEVELOPMENT) + yield test def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True): + for test in _AllFoundYamlTests(use_short_run_name=True): + yield test + + +def AllDarwinFrameworkToolYamlTests(): + unsupported_tests = _GetDarwinFrameworkToolUnsupportedTests() + for test in _AllFoundYamlTests(use_short_run_name=True): + if test.run_name in unsupported_tests: + test.tags.add(TestTag.IN_DEVELOPMENT) + yield test diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 6894693a0fa500..4061870a074c5e 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -160,8 +160,11 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, # Figures out selected test that match the given name(s) if runtime == TestRunTime.CHIP_REPL_PYTHON: all_tests = [test for test in chiptest.AllReplYamlTests()] - elif runtime == TestRunTime.CHIP_TOOL_PYTHON and os.path.basename(chip_tool) != "darwin-framework-tool": - all_tests = [test for test in chiptest.AllChipToolYamlTests()] + elif runtime == TestRunTime.CHIP_TOOL_PYTHON: + if os.path.basename(chip_tool) == "darwin-framework-tool": + all_tests = [test for test in chiptest.AllDarwinFrameworkToolYamlTests()] + else: + all_tests = [test for test in chiptest.AllChipToolYamlTests()] else: all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)]