From 9e24b3ad980c38173a57a89acc61aed65bfcf3cf Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Wed, 13 Sep 2023 14:20:56 +0200 Subject: [PATCH] [MatterYamlTests][darwin-framework-tool] Use a list of tests to run from scripts/tests/chiptest/__init__.py if chip_tool_python is used in conjunction with darwinframeworktool.py --- .github/workflows/darwin-tests.yaml | 1 + scripts/tests/chiptest/__init__.py | 65 +++++++++++++++++++++-- scripts/tests/chiptest/test_definition.py | 58 ++++++++++---------- scripts/tests/run_test_suite.py | 28 +++++----- 4 files changed, 104 insertions(+), 48 deletions(-) diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml index d4db7d1ce0f289..288166dfb87130 100644 --- a/.github/workflows/darwin-tests.yaml +++ b/.github/workflows/darwin-tests.yaml @@ -106,6 +106,7 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ + --runner darwin_framework_tool_python \ --chip-tool ./out/darwin-x64-darwin-framework-tool-${BUILD_VARIANT_FRAMEWORK_TOOL}/darwin-framework-tool \ --target-skip-glob '{TestAccessControlConstraints}' \ run \ diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 378e3df2e02669..d449f235a51fab 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -160,6 +160,56 @@ def _GetInDevelopmentTests() -> Set[str]: } +def _GetDarwinFrameworkToolUnsupportedTests() -> Set[str]: + """Tests that fail in darwin-framework-tool for some reason""" + return { + "DL_LockUnlock", # Events error. + "Test_AddNewFabricFromExistingFabric", # darwin-framework-tool does not support the GetCommissionerRootCertificate command. + "TestAttributesById", + "TestBasicInformation", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "TestClusterComplexTypes", + "TestClusterMultiFabric", + "TestCommandsById", # Test Failure : The test expects the "UNSUPPORTED_ENDPOINT" error but the "FAILURE" error occured. + "TestCommissioningWindow", + "TestDiagnosticLogs", # darwin-framework-tool does not implement a BDXTransferServerDelegate + "TestDiscovery", # darwin-framework-tool does not support dns-sd commands. + "TestEvents", + "TestEventsById", + "TestGroupMessaging", # darwin-framework-tool does not support group commands. + "TestReadNoneSubscribeNone", # darwin-framework-tool does not supports those commands. + + "Test_TC_ACE_1_6", # darwin-framework-tool does not support group commands. + "Test_TC_ACL_2_5", # Events error. + "Test_TC_ACL_2_6", # Events error. + "Test_TC_ACL_2_7", # Events error. + "Test_TC_ACL_2_8", # Events error. + "Test_TC_ACL_2_9", # Events error. + "Test_TC_ACL_2_10", # Events error. + "Test_TC_BINFO_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_BINFO_2_2", # Events error. + "Test_TC_BRBINFO_2_1", # Test Failure : The test expects the "UNSUPPORTED_ATTRIBUTE" error but the "FAILURE" error occured. + "Test_TC_DGEN_2_1", # minValue: 1 <-- The response value (0) should be greater or equal to the constraint but 0 < 1 + "Test_TC_DGGEN_2_3", # Events error. + "Test_TC_DRLK_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_DGTHREAD_2_1", + "Test_TC_DGTHREAD_2_2", + "Test_TC_DGTHREAD_2_3", + "Test_TC_DGTHREAD_2_4", + "Test_TC_FLABEL_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_GRPKEY_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_LCFG_2_1", # Test Failure : The test expects the "UNSUPPORTED_WRITE" error but the "FAILURE" error occured + "Test_TC_OPCREDS_3_7", # darwin-framework-tool does not support the GetCommissionerRootCertificate command. + "Test_TC_OPSTATE_2_4", # Events error. + "Test_TC_SMOKECO_2_2", # Events error. + "Test_TC_SMOKECO_2_3", # Events error. + "Test_TC_SMOKECO_2_4", # Events error. + "Test_TC_SMOKECO_2_5", # Events error. + "Test_TC_SMOKECO_2_6", # Events error. + "Test_TC_SC_4_1", # darwin-framework-tool does not support dns-sd commands. + "Test_TC_SC_5_2", # darwin-framework-tool does not support group commands. + } + + def _GetChipReplUnsupportedTests() -> Set[str]: """Tests that fail in chip-repl for some reason""" return { @@ -284,7 +334,7 @@ def tests_with_command(chip_tool: str, is_manual: bool): ) -def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool): +def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, dft_unsupported: bool, use_short_run_name: bool): """ use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml" """ @@ -294,6 +344,7 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short extra_slow_tests = _GetExtraSlowTests() in_development_tests = _GetInDevelopmentTests() chip_repl_unsupported_tests = _GetChipReplUnsupportedTests() + dft_unsupported_tests = _GetDarwinFrameworkToolUnsupportedTests() purposeful_failure_tests = _GetPurposefulFailureTests() for path in _AllYamlTests(): @@ -327,6 +378,9 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short else: run_name = str(path) + if dft_unsupported and run_name in dft_unsupported_tests: + tags.add(TestTag.IN_DEVELOPMENT) + yield TestDefinition( run_name=run_name, name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" @@ -336,12 +390,17 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short def AllReplYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, dft_unsupported=False, use_short_run_name=False): yield test def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, dft_unsupported=False, use_short_run_name=True): + yield test + + +def AllDarwinFrameworkToolYamlTests(): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, dft_unsupported=True, use_short_run_name=True): yield test diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py index da1b56afd919e1..19f0a7dcb74b0b 100644 --- a/scripts/tests/chiptest/test_definition.py +++ b/scripts/tests/chiptest/test_definition.py @@ -251,10 +251,9 @@ def to_s(self): class TestRunTime(Enum): - CHIP_TOOL_BUILTIN = auto() # run via chip-tool built-in test commands CHIP_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool + DARWIN_FRAMEWORK_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool CHIP_REPL_PYTHON = auto() # use the python yaml test runner - DARWIN_FRAMEWORK_TOOL_BUILTIN = auto() # run via darwin-framework-tool built-in test commands @dataclass @@ -281,7 +280,7 @@ def tags_str(self) -> str: return ", ".join([t.to_s() for t in self.tags]) def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, - timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_BUILTIN): + timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_PYTHON): """ Executes the given test case using the provided runner for execution. """ @@ -333,8 +332,6 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, # so it will be commissionable again. app.factoryReset() - tool_cmd = paths.chip_tool if test_runtime != TestRunTime.CHIP_TOOL_PYTHON else paths.chip_tool_with_python_cmd - if dry_run: tool_storage_dir = None tool_storage_args = [] @@ -350,38 +347,37 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, app.start() setupCode = app.setupCode - pairing_cmd = tool_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode] - test_cmd = tool_cmd + ['tests', self.run_name] + ['--PICS', pics_file] - if test_runtime == TestRunTime.CHIP_TOOL_PYTHON: + if test_runtime == TestRunTime.CHIP_REPL_PYTHON: + chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd + python_cmd = chip_repl_yaml_tester_cmd + \ + ['--setup-code', setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file] + if dry_run: + logging.info(" ".join(python_cmd)) + else: + runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER', + dependencies=[apps_register], timeout_seconds=timeout_seconds) + else: + pairing_cmd = paths.chip_tool_with_python_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode] + test_cmd = paths.chip_tool_with_python_cmd + ['tests', self.run_name] + ['--PICS', pics_file] server_args = ['--server_path', paths.chip_tool[-1]] + \ ['--server_arguments', 'interactive server' + (' ' if len(tool_storage_args) else '') + ' '.join(tool_storage_args)] pairing_cmd += server_args test_cmd += server_args - elif test_runtime == TestRunTime.CHIP_TOOL_BUILTIN: - pairing_cmd += tool_storage_args - test_cmd += tool_storage_args - - if dry_run: - # Some of our command arguments have spaces in them, so if we are - # trying to log commands people can run we should quote those. - def quoter(arg): return f"'{arg}'" if ' ' in arg else arg - logging.info(" ".join(map(quoter, pairing_cmd))) - logging.info(" ".join(map(quoter, test_cmd))) - elif test_runtime == TestRunTime.CHIP_REPL_PYTHON: - chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd - python_cmd = chip_repl_yaml_tester_cmd + \ - ['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file] - runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER', - dependencies=[apps_register], timeout_seconds=timeout_seconds) - else: - runner.RunSubprocess(pairing_cmd, - name='PAIR', dependencies=[apps_register]) - runner.RunSubprocess( - test_cmd, - name='TEST', dependencies=[apps_register], - timeout_seconds=timeout_seconds) + if dry_run: + # Some of our command arguments have spaces in them, so if we are + # trying to log commands people can run we should quote those. + def quoter(arg): return f"'{arg}'" if ' ' in arg else arg + logging.info(" ".join(map(quoter, pairing_cmd))) + logging.info(" ".join(map(quoter, test_cmd))) + else: + runner.RunSubprocess(pairing_cmd, + name='PAIR', dependencies=[apps_register]) + runner.RunSubprocess( + test_cmd, + name='TEST', dependencies=[apps_register], + timeout_seconds=timeout_seconds) except Exception: logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!") diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 03773e4fb88f84..d4d94b28bf858c 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -123,8 +123,8 @@ class RunContext: ) @click.option( '--runner', - type=click.Choice(['codegen', 'chip_repl_python', 'chip_tool_python'], case_sensitive=False), - default='codegen', + type=click.Choice(['chip_repl_python', 'chip_tool_python', 'darwin_framework_tool_python'], case_sensitive=False), + default='chip_tool_python', help='Run YAML tests using the specified runner.') @click.option( '--chip-tool', @@ -138,18 +138,18 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, log_fmt = '%(levelname)-7s %(message)s' coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt) - runtime = TestRunTime.CHIP_TOOL_BUILTIN + runtime = TestRunTime.CHIP_TOOL_PYTHON if runner == 'chip_repl_python': runtime = TestRunTime.CHIP_REPL_PYTHON - elif runner == 'chip_tool_python': - runtime = TestRunTime.CHIP_TOOL_PYTHON - elif chip_tool is not None and os.path.basename(chip_tool) == "darwin-framework-tool": - runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_BUILTIN + elif runner == 'darwin_framework_tool_python': + runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON if chip_tool is None and not runtime == TestRunTime.CHIP_REPL_PYTHON: - # non yaml tests REQUIRE chip-tool. Yaml tests should not require chip-tool paths_finder = PathsFinder() - chip_tool = paths_finder.get('chip-tool') + if runtime == TestRunTime.CHIP_TOOL_PYTHON: + chip_tool = paths_finder.get('chip-tool') + else: + chip_tool = paths_finder.get('darwin-framework-tool') if include_tags: include_tags = set([TestTag.__members__[t] for t in include_tags]) @@ -160,10 +160,10 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, # Figures out selected test that match the given name(s) if runtime == TestRunTime.CHIP_REPL_PYTHON: all_tests = [test for test in chiptest.AllReplYamlTests()] - elif runtime == TestRunTime.CHIP_TOOL_PYTHON and os.path.basename(chip_tool) != "darwin-framework-tool": - all_tests = [test for test in chiptest.AllChipToolYamlTests()] + elif runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON: + all_tests = [test for test in chiptest.AllDarwinFrameworkToolYamlTests()] else: - all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)] + all_tests = [test for test in chiptest.AllChipToolYamlTests()] tests = all_tests @@ -178,7 +178,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, TestTag.PURPOSEFUL_FAILURE, } - if runtime != TestRunTime.CHIP_TOOL_PYTHON: + if runtime == TestRunTime.CHIP_REPL_PYTHON: exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY) if 'all' not in target: @@ -319,7 +319,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o chip_repl_yaml_tester = paths_finder.get('yamltest_with_chip_repl_tester.py') if chip_tool_with_python is None: - if context.obj.chip_tool and os.path.basename(context.obj.chip_tool) == "darwin-framework-tool": + if context.obj.runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON: chip_tool_with_python = paths_finder.get('darwinframeworktool.py') else: chip_tool_with_python = paths_finder.get('chiptool.py')