Actually enable our YAML tests in the python-calls-chip-tool runner.

We were now getting the right test list, but for each test we were not finding
the test file.  And the harness did not treat that as an error.

scripts/tests/chiptest/__init__.py

@@ -47,6 +47,9 @@ class ManualTest:
     "PICS_Example.yaml",
     "Response_Example.yaml",
     "Test_Example.yaml",
+    "Test_Example_1.yaml",
+    "Test_Example_2.yaml",
+    "Test_Example_3.yaml",
 }
 
 
@@ -243,7 +246,10 @@ def tests_with_command(chip_tool: str, is_manual: bool):
         )
 
 
-def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
+def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
+    """
+    use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
+    """
     manual_tests = _GetManualTests()
     flaky_tests = _GetFlakyTests()
     slow_tests = _GetSlowTests()
@@ -270,21 +276,26 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
         if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests:
             tags.add(TestTag.IN_DEVELOPMENT)
 
+        if use_short_run_name:
+            run_name = path.stem  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
+        else:
+            run_name = str(path)
+
         yield TestDefinition(
-            run_name=str(path),
+            run_name=run_name,
             name=path.stem,  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
             target=target_for_name(path.name),
             tags=tags,
         )
 
 
 def AllReplYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
         yield test
 
 
 def AllChipToolYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
         yield test
 
 

scripts/tests/yaml/runner.py

@@ -256,7 +256,10 @@ def runner_base(ctx, configuration_directory: str, test_name: str, configuration
     tests_finder = TestsFinder(configuration_directory, configuration_name)
 
     parser_config = TestParserConfig(pics, specifications, kwargs)
-    parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger())
+    test_list = tests_finder.get(test_name)
+    if len(test_list) == 0:
+        raise Exception(f"No tests found for test name '{test_name}'")
+    parser_builder_config = TestParserBuilderConfig(test_list, parser_config, hooks=TestParserLogger())
     parser_builder_config.options.stop_on_error = stop_on_error
     while ctx:
         ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters)

src/app/tests/suites/TestClusterMultiFabric.yaml

@@ -284,8 +284,7 @@ tests:
           #
           #      TODO: This can be fixed using the `saveAs` function.
           #
-          value:
-              [
+          value: [
                   {
                       FabricIndex: 1,
                       fabricSensitiveInt8u: 33,
@@ -328,39 +327,27 @@ tests:
                   },
                   {
                       FabricIndex: 2,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
                   {
                       FabricIndex: 2,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
               ]
 
@@ -371,43 +358,30 @@ tests:
       fabricFiltered: false
       attribute: "list_fabric_scoped"
       response:
-          value:
-              [
+          value: [
                   {
                       FabricIndex: 1,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
                   {
                       FabricIndex: 1,
-                      fabricSensitiveInt8u: 0,
+                      # These should actually be missing, not null, but right
+                      # now our harness treats those the same, and we have no
+                      # way to indicate "missing" in the YAML.
+                      fabricSensitiveInt8u: null,
+                      optionalFabricSensitiveInt8u: null,
                       nullableFabricSensitiveInt8u: null,
-                      fabricSensitiveCharString: "",
-                      fabricSensitiveStruct:
-                          {
-                              a: 0,
-                              b: false,
-                              c: 0,
-                              d: "",
-                              e: "",
-                              f: 0,
-                              g: 0,
-                              h: 0,
-                          },
-                      fabricSensitiveInt8uList: [],
+                      fabricSensitiveCharString: null,
+                      fabricSensitiveStruct: null,
+                      fabricSensitiveInt8uList: null,
                   },
                   {
                       FabricIndex: 2,

src/app/tests/suites/TestEventsById.yaml

@@ -48,7 +48,8 @@ tests:
                 value: ReadRequestMessage.Cluster
               - name: "EventId"
                 value: ReadRequestMessage.Event
-      response: []
+      response:
+          error: UNSUPPORTED_ENDPOINT
 
     - label:
           "Read Request Message with a path that indicates a specific cluster
@@ -62,11 +63,12 @@ tests:
                 value: UnsupportedCluster
               - name: "EventId"
                 value: ReadRequestMessage.Event
-      response: []
+      response:
+          error: UNSUPPORTED_CLUSTER
 
     - label:
-          "Read Request Message with a path that indicates a specific endpoint
-          that is unsupported"
+          "Read Request Message with a path that indicates a specific event that
+          is unsupported"
       cluster: "AnyCommands"
       command: "ReadEventById"
       endpoint: ReadRequestMessage.EndPoint
@@ -256,9 +258,5 @@ tests:
               - name: "EventId"
                 value: ReadRequestMessage.EndPoint
       response:
-          - values:
-                - value: { arg1: 1, arg2: 2, arg3: true }
-          - values:
-                - value: { arg1: 3, arg2: 1, arg3: false }
-          - values:
-                - value: { arg1: 4, arg2: 3, arg3: true }
+          # Not a valid event request path.
+          error: FAILURE