From cd434784efa61ae3d221a5c407989cc56b22ee20 Mon Sep 17 00:00:00 2001 From: Boris Zbarsky Date: Wed, 6 Sep 2023 12:14:48 -0400 Subject: [PATCH] Actually enable our YAML tests in the python-calls-chip-tool runner. We were now getting the right test list, but for each test we were not finding the test file. And the harness did not treat that as an error. --- scripts/tests/chiptest/__init__.py | 16 +++- scripts/tests/yaml/runner.py | 5 +- .../tests/suites/TestClusterMultiFabric.yaml | 94 +++++++------------ src/app/tests/suites/TestEventsById.yaml | 18 ++-- 4 files changed, 58 insertions(+), 75 deletions(-) diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index ce312d492beabd..3bd1a1a95df1a4 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -243,7 +243,10 @@ def tests_with_command(chip_tool: str, is_manual: bool): ) -def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): +def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool): + """ + use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml" + """ manual_tests = _GetManualTests() flaky_tests = _GetFlakyTests() slow_tests = _GetSlowTests() @@ -270,8 +273,13 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests: tags.add(TestTag.IN_DEVELOPMENT) + if use_short_run_name: + run_name = path.stem # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" + else: + run_name = str(path) + yield TestDefinition( - run_name=str(path), + run_name=run_name, name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2" target=target_for_name(path.name), tags=tags, @@ -279,12 +287,12 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool): def AllReplYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False): yield test def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True): yield test diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/yaml/runner.py index 04f30401eb1b02..04a1f9453d3b98 100755 --- a/scripts/tests/yaml/runner.py +++ b/scripts/tests/yaml/runner.py @@ -256,7 +256,10 @@ def runner_base(ctx, configuration_directory: str, test_name: str, configuration tests_finder = TestsFinder(configuration_directory, configuration_name) parser_config = TestParserConfig(pics, specifications, kwargs) - parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger()) + test_list = tests_finder.get(test_name) + if len(test_list) == 0: + raise Exception(f"No tests found for test name '{test_name}'") + parser_builder_config = TestParserBuilderConfig(test_list, parser_config, hooks=TestParserLogger()) parser_builder_config.options.stop_on_error = stop_on_error while ctx: ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters) diff --git a/src/app/tests/suites/TestClusterMultiFabric.yaml b/src/app/tests/suites/TestClusterMultiFabric.yaml index 97b38ee103df85..7ca177dcbb67aa 100644 --- a/src/app/tests/suites/TestClusterMultiFabric.yaml +++ b/src/app/tests/suites/TestClusterMultiFabric.yaml @@ -284,8 +284,7 @@ tests: # # TODO: This can be fixed using the `saveAs` function. # - value: - [ + value: [ { FabricIndex: 1, fabricSensitiveInt8u: 33, @@ -328,39 +327,27 @@ tests: }, { FabricIndex: 2, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 2, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, ] @@ -371,43 +358,30 @@ tests: fabricFiltered: false attribute: "list_fabric_scoped" response: - value: - [ + value: [ { FabricIndex: 1, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 1, - fabricSensitiveInt8u: 0, + # These should actually be missing, not null, but right + # now our harness treats those the same, and we have no + # way to indicate "missing" in the YAML. + fabricSensitiveInt8u: null, + optionalFabricSensitiveInt8u: null, nullableFabricSensitiveInt8u: null, - fabricSensitiveCharString: "", - fabricSensitiveStruct: - { - a: 0, - b: false, - c: 0, - d: "", - e: "", - f: 0, - g: 0, - h: 0, - }, - fabricSensitiveInt8uList: [], + fabricSensitiveCharString: null, + fabricSensitiveStruct: null, + fabricSensitiveInt8uList: null, }, { FabricIndex: 2, diff --git a/src/app/tests/suites/TestEventsById.yaml b/src/app/tests/suites/TestEventsById.yaml index 29224f92e3e7ac..fa772eacb1d56e 100644 --- a/src/app/tests/suites/TestEventsById.yaml +++ b/src/app/tests/suites/TestEventsById.yaml @@ -48,7 +48,8 @@ tests: value: ReadRequestMessage.Cluster - name: "EventId" value: ReadRequestMessage.Event - response: [] + response: + error: UNSUPPORTED_ENDPOINT - label: "Read Request Message with a path that indicates a specific cluster @@ -62,11 +63,12 @@ tests: value: UnsupportedCluster - name: "EventId" value: ReadRequestMessage.Event - response: [] + response: + error: UNSUPPORTED_CLUSTER - label: - "Read Request Message with a path that indicates a specific endpoint - that is unsupported" + "Read Request Message with a path that indicates a specific event that + is unsupported" cluster: "AnyCommands" command: "ReadEventById" endpoint: ReadRequestMessage.EndPoint @@ -256,9 +258,5 @@ tests: - name: "EventId" value: ReadRequestMessage.EndPoint response: - - values: - - value: { arg1: 1, arg2: 2, arg3: true } - - values: - - value: { arg1: 3, arg2: 1, arg3: false } - - values: - - value: { arg1: 4, arg2: 3, arg3: true } + # Not a valid event request path. + error: FAILURE