diff --git a/.github/workflows/darwin-tests.yaml b/.github/workflows/darwin-tests.yaml index d4db7d1ce0f289..288166dfb87130 100644 --- a/.github/workflows/darwin-tests.yaml +++ b/.github/workflows/darwin-tests.yaml @@ -106,6 +106,7 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ + --runner darwin_framework_tool_python \ --chip-tool ./out/darwin-x64-darwin-framework-tool-${BUILD_VARIANT_FRAMEWORK_TOOL}/darwin-framework-tool \ --target-skip-glob '{TestAccessControlConstraints}' \ run \ diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py index a7dd32b7d43a85..cd77147fff40d6 100644 --- a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py +++ b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py @@ -322,7 +322,7 @@ def run(self, specs, value, cluster_name: str, typename: str, array: bool): provided_field_name = provided_field_name[0].lower( ) + provided_field_name[1:] - if provided_field_name in value: + if provided_field_name in value and provided_field_name != field_name: value[field_name] = self.run( specs, value[provided_field_name], diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 378e3df2e02669..dba11f15d44c1e 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -160,6 +160,60 @@ def _GetInDevelopmentTests() -> Set[str]: } +def _GetDarwinFrameworkToolUnsupportedTests() -> Set[str]: + """Tests that fail in darwin-framework-tool for some reason""" + return { + "DL_LockUnlock", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_AddNewFabricFromExistingFabric", # darwin-framework-tool does not support the GetCommissionerRootCertificate command. + # The name of the arguments once converted differs for chip-tool and darwin-framework-tool (attribute-ids vs attribute-id. See #31934) + "TestAttributesById", + "TestBasicInformation", # darwin-framework-tool does not support writing readonly attributes by name + "TestClusterComplexTypes", # Darwin framework has no way to represent a present but null optional nullable field. + # When reading TestFabricScoped in TestClusterMultiFabric, the result differs because of missing fields that have been declared in the YAML step with null value to workaround some limitation of the test harness (#29110) + "TestClusterMultiFabric", + "TestCommandsById", # darwin-framework-tool does not support writing readonly attributes by name + "TestDiagnosticLogs", # darwin-framework-tool does not implement a BDXTransferServerDelegate + "TestDiscovery", # darwin-framework-tool does not support dns-sd commands. + "TestEvents", # darwin-framework-tool does not currently support reading or subscribing to Events + "TestEventsById", # darwin-framework-tool does not currently support reading or subscribing to Events + "TestGroupMessaging", # darwin-framework-tool does not support group commands. + "TestIcdManagementCluster", # darwin-framework-tool does not support ICD registration + "TestUnitTestingClusterMei", # darwin-framework-tool does not currently support reading or subscribing to Events + "TestReadNoneSubscribeNone", # darwin-framework-tool does not supports those commands. + + "Test_TC_ACE_1_6", # darwin-framework-tool does not support group commands. + "Test_TC_ACL_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_ACL_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_ACL_2_7", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_ACL_2_8", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_ACL_2_9", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_ACL_2_10", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_BINFO_2_1", # darwin-framework-tool does not support writing readonly attributes by name + "Test_TC_BINFO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events + # The name of the arguments once converted differs for chip-tool and darwin-framework-tool (attribute-ids vs attribute-id. See #31934) + "Test_TC_BRBINFO_2_1", + "Test_TC_DGGEN_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_DRLK_2_1", # darwin-framework-tool does not support writing readonly attributes by name + "Test_TC_DGTHREAD_2_1", # Thread Network Diagnostics is not implemented under darwin. + "Test_TC_DGTHREAD_2_2", # Thread Network Diagnostics is not implemented under darwin. + "Test_TC_DGTHREAD_2_3", # Thread Network Diagnostics is not implemented under darwin. + "Test_TC_DGTHREAD_2_4", # Thread Network Diagnostics is not implemented under darwin. + "Test_TC_FLABEL_2_1", # darwin-framework-tool does not support writing readonly attributes by name + "Test_TC_GRPKEY_2_1", # darwin-framework-tool does not support writing readonly attributes by name + "Test_TC_LCFG_2_1", # darwin-framework-tool does not support writing readonly attributes by name + "Test_TC_OPCREDS_3_7", # darwin-framework-tool does not support the GetCommissionerRootCertificate command. + "Test_TC_OPSTATE_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_SMOKECO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_SMOKECO_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_SMOKECO_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_SMOKECO_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_SMOKECO_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events + "Test_TC_SC_4_1", # darwin-framework-tool does not support dns-sd commands. + "Test_TC_SC_5_2", # darwin-framework-tool does not support group commands. + "Test_TC_S_2_3", # darwin-framework-tool does not support group commands. + } + + def _GetChipReplUnsupportedTests() -> Set[str]: """Tests that fail in chip-repl for some reason""" return { @@ -284,7 +338,7 @@ def tests_with_command(chip_tool: str, is_manual: bool): ) -def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool): +def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, treat_dft_unsupported_as_in_development: bool, use_short_run_name: bool): """ use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml" """ @@ -294,6 +348,7 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short extra_slow_tests = _GetExtraSlowTests() in_development_tests = _GetInDevelopmentTests() chip_repl_unsupported_tests = _GetChipReplUnsupportedTests() + treat_dft_unsupported_as_in_development_tests = _GetDarwinFrameworkToolUnsupportedTests() purposeful_failure_tests = _GetPurposefulFailureTests() for path in _AllYamlTests(): @@ -327,6 +382,9 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short else: run_name = str(path) + if treat_dft_unsupported_as_in_development and run_name in treat_dft_unsupported_as_in_development_tests: + tags.add(TestTag.IN_DEVELOPMENT) + yield TestDefinition( run_name=run_name, name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" @@ -336,12 +394,17 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short def AllReplYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, treat_dft_unsupported_as_in_development=False, use_short_run_name=False): yield test def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, use_short_run_name=True): + yield test + + +def AllDarwinFrameworkToolYamlTests(): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=True, use_short_run_name=True): yield test diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py index da1b56afd919e1..19f0a7dcb74b0b 100644 --- a/scripts/tests/chiptest/test_definition.py +++ b/scripts/tests/chiptest/test_definition.py @@ -251,10 +251,9 @@ def to_s(self): class TestRunTime(Enum): - CHIP_TOOL_BUILTIN = auto() # run via chip-tool built-in test commands CHIP_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool + DARWIN_FRAMEWORK_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool CHIP_REPL_PYTHON = auto() # use the python yaml test runner - DARWIN_FRAMEWORK_TOOL_BUILTIN = auto() # run via darwin-framework-tool built-in test commands @dataclass @@ -281,7 +280,7 @@ def tags_str(self) -> str: return ", ".join([t.to_s() for t in self.tags]) def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, - timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_BUILTIN): + timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_PYTHON): """ Executes the given test case using the provided runner for execution. """ @@ -333,8 +332,6 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, # so it will be commissionable again. app.factoryReset() - tool_cmd = paths.chip_tool if test_runtime != TestRunTime.CHIP_TOOL_PYTHON else paths.chip_tool_with_python_cmd - if dry_run: tool_storage_dir = None tool_storage_args = [] @@ -350,38 +347,37 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, app.start() setupCode = app.setupCode - pairing_cmd = tool_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode] - test_cmd = tool_cmd + ['tests', self.run_name] + ['--PICS', pics_file] - if test_runtime == TestRunTime.CHIP_TOOL_PYTHON: + if test_runtime == TestRunTime.CHIP_REPL_PYTHON: + chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd + python_cmd = chip_repl_yaml_tester_cmd + \ + ['--setup-code', setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file] + if dry_run: + logging.info(" ".join(python_cmd)) + else: + runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER', + dependencies=[apps_register], timeout_seconds=timeout_seconds) + else: + pairing_cmd = paths.chip_tool_with_python_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode] + test_cmd = paths.chip_tool_with_python_cmd + ['tests', self.run_name] + ['--PICS', pics_file] server_args = ['--server_path', paths.chip_tool[-1]] + \ ['--server_arguments', 'interactive server' + (' ' if len(tool_storage_args) else '') + ' '.join(tool_storage_args)] pairing_cmd += server_args test_cmd += server_args - elif test_runtime == TestRunTime.CHIP_TOOL_BUILTIN: - pairing_cmd += tool_storage_args - test_cmd += tool_storage_args - - if dry_run: - # Some of our command arguments have spaces in them, so if we are - # trying to log commands people can run we should quote those. - def quoter(arg): return f"'{arg}'" if ' ' in arg else arg - logging.info(" ".join(map(quoter, pairing_cmd))) - logging.info(" ".join(map(quoter, test_cmd))) - elif test_runtime == TestRunTime.CHIP_REPL_PYTHON: - chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd - python_cmd = chip_repl_yaml_tester_cmd + \ - ['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file] - runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER', - dependencies=[apps_register], timeout_seconds=timeout_seconds) - else: - runner.RunSubprocess(pairing_cmd, - name='PAIR', dependencies=[apps_register]) - runner.RunSubprocess( - test_cmd, - name='TEST', dependencies=[apps_register], - timeout_seconds=timeout_seconds) + if dry_run: + # Some of our command arguments have spaces in them, so if we are + # trying to log commands people can run we should quote those. + def quoter(arg): return f"'{arg}'" if ' ' in arg else arg + logging.info(" ".join(map(quoter, pairing_cmd))) + logging.info(" ".join(map(quoter, test_cmd))) + else: + runner.RunSubprocess(pairing_cmd, + name='PAIR', dependencies=[apps_register]) + runner.RunSubprocess( + test_cmd, + name='TEST', dependencies=[apps_register], + timeout_seconds=timeout_seconds) except Exception: logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!") diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index 03773e4fb88f84..d4d94b28bf858c 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -123,8 +123,8 @@ class RunContext: ) @click.option( '--runner', - type=click.Choice(['codegen', 'chip_repl_python', 'chip_tool_python'], case_sensitive=False), - default='codegen', + type=click.Choice(['chip_repl_python', 'chip_tool_python', 'darwin_framework_tool_python'], case_sensitive=False), + default='chip_tool_python', help='Run YAML tests using the specified runner.') @click.option( '--chip-tool', @@ -138,18 +138,18 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, log_fmt = '%(levelname)-7s %(message)s' coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt) - runtime = TestRunTime.CHIP_TOOL_BUILTIN + runtime = TestRunTime.CHIP_TOOL_PYTHON if runner == 'chip_repl_python': runtime = TestRunTime.CHIP_REPL_PYTHON - elif runner == 'chip_tool_python': - runtime = TestRunTime.CHIP_TOOL_PYTHON - elif chip_tool is not None and os.path.basename(chip_tool) == "darwin-framework-tool": - runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_BUILTIN + elif runner == 'darwin_framework_tool_python': + runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON if chip_tool is None and not runtime == TestRunTime.CHIP_REPL_PYTHON: - # non yaml tests REQUIRE chip-tool. Yaml tests should not require chip-tool paths_finder = PathsFinder() - chip_tool = paths_finder.get('chip-tool') + if runtime == TestRunTime.CHIP_TOOL_PYTHON: + chip_tool = paths_finder.get('chip-tool') + else: + chip_tool = paths_finder.get('darwin-framework-tool') if include_tags: include_tags = set([TestTag.__members__[t] for t in include_tags]) @@ -160,10 +160,10 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, # Figures out selected test that match the given name(s) if runtime == TestRunTime.CHIP_REPL_PYTHON: all_tests = [test for test in chiptest.AllReplYamlTests()] - elif runtime == TestRunTime.CHIP_TOOL_PYTHON and os.path.basename(chip_tool) != "darwin-framework-tool": - all_tests = [test for test in chiptest.AllChipToolYamlTests()] + elif runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON: + all_tests = [test for test in chiptest.AllDarwinFrameworkToolYamlTests()] else: - all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)] + all_tests = [test for test in chiptest.AllChipToolYamlTests()] tests = all_tests @@ -178,7 +178,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, TestTag.PURPOSEFUL_FAILURE, } - if runtime != TestRunTime.CHIP_TOOL_PYTHON: + if runtime == TestRunTime.CHIP_REPL_PYTHON: exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY) if 'all' not in target: @@ -319,7 +319,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o chip_repl_yaml_tester = paths_finder.get('yamltest_with_chip_repl_tester.py') if chip_tool_with_python is None: - if context.obj.chip_tool and os.path.basename(context.obj.chip_tool) == "darwin-framework-tool": + if context.obj.runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON: chip_tool_with_python = paths_finder.get('darwinframeworktool.py') else: chip_tool_with_python = paths_finder.get('chiptool.py')