From a050392570fcc7e6117b53d513afdf9f9b61b9c5 Mon Sep 17 00:00:00 2001 From: Boris Zbarsky Date: Wed, 6 Sep 2023 12:14:48 -0400 Subject: [PATCH] Actually enable our YAML tests in the python-calls-chip-tool runner. We were now getting the right test list, but for each test we were not finding the test file. And the harness did not treat that as an error. --- scripts/tests/chiptest/__init__.py | 16 +++- scripts/tests/yaml/runner.py | 5 +- .../tests/suites/TestClusterMultiFabric.yaml | 94 +++++++------------ 3 files changed, 50 insertions(+), 65 deletions(-) diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index ce312d492beabd..3bd1a1a95df1a4 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -243,7 +243,10 @@ def tests_with_command(chip_tool: str, is_manual: bool): ) -def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): +def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool): + """ + use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml" + """ manual_tests = _GetManualTests() flaky_tests = _GetFlakyTests() slow_tests = _GetSlowTests() @@ -270,8 +273,13 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests: tags.add(TestTag.IN_DEVELOPMENT) + if use_short_run_name: + run_name = path.stem # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" + else: + run_name = str(path) + yield TestDefinition( - run_name=str(path), + run_name=run_name, name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" target=target_for_name(path.name), tags=tags, @@ -279,12 +287,12 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): def AllReplYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False): yield test def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True): yield test diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/yaml/runner.py index 04f30401eb1b02..04a1f9453d3b98 100755 --- a/scripts/tests/yaml/runner.py +++ b/scripts/tests/yaml/runner.py @@ -256,7 +256,10 @@ def runner_base(ctx, configuration_directory: str, test_name: str, configuration tests_finder = TestsFinder(configuration_directory, configuration_name) parser_config = TestParserConfig(pics, specifications, kwargs) - parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger()) + test_list = tests_finder.get(test_name) + if len(test_list) == 0: + raise Exception(f"No tests found for test name '{test_name}'") + parser_builder_config = TestParserBuilderConfig(test_list, parser_config, hooks=TestParserLogger()) parser_builder_config.options.stop_on_error = stop_on_error while ctx: ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters) diff --git a/src/app/tests/suites/TestClusterMultiFabric.yaml b/src/app/tests/suites/TestClusterMultiFabric.yaml index 97b38ee103df85..7ca177dcbb67aa 100644 --- a/src/app/tests/suites/TestClusterMultiFabric.yaml +++ b/src/app/tests/suites/TestClusterMultiFabric.yaml @@ -284,8 +284,7 @@ tests: # # TODO: This can be fixed using the `saveAs` function. # - value: - [ + value: [ { FabricIndex: 1, fabricSensitiveInt8u: 33, @@ -328,39 +327,27 @@ tests: }, { FabricIndex: 2, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 2, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, ] @@ -371,43 +358,30 @@ tests: fabricFiltered: false attribute: "list_fabric_scoped" response: - value: - [ + value: [ { FabricIndex: 1, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 1, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 2,