diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser_builder.py b/scripts/py_matter_yamltests/matter_yamltests/parser_builder.py
index 819406f2d43cfd..6be0e0b01c61e2 100644
--- a/scripts/py_matter_yamltests/matter_yamltests/parser_builder.py
+++ b/scripts/py_matter_yamltests/matter_yamltests/parser_builder.py
@@ -65,6 +65,8 @@ class TestParserBuilder:
 
     def __init__(self, config: TestParserBuilderConfig = TestParserBuilderConfig()):
         self.__tests = copy.copy(config.tests)
+        if len(self.__tests) == 0:
+            raise Exception(f'{config} did not provide any tests')
         self.__config = config
         self.__duration = 0
         self.done = False
diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py
index ce312d492beabd..3bd1a1a95df1a4 100644
--- a/scripts/tests/chiptest/__init__.py
+++ b/scripts/tests/chiptest/__init__.py
@@ -243,7 +243,10 @@ def tests_with_command(chip_tool: str, is_manual: bool):
         )
 
 
-def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
+def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
+    """
+    use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
+    """
     manual_tests = _GetManualTests()
     flaky_tests = _GetFlakyTests()
     slow_tests = _GetSlowTests()
@@ -270,8 +273,13 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
         if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests:
             tags.add(TestTag.IN_DEVELOPMENT)
 
+        if use_short_run_name:
+            run_name = path.stem  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
+        else:
+            run_name = str(path)
+
         yield TestDefinition(
-            run_name=str(path),
+            run_name=run_name,
             name=path.stem,  # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
             target=target_for_name(path.name),
             tags=tags,
@@ -279,12 +287,12 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
 
 
 def AllReplYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
         yield test
 
 
 def AllChipToolYamlTests():
-    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False):
+    for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
         yield test
 
 
diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/yaml/runner.py
index 04f30401eb1b02..04a1f9453d3b98 100755
--- a/scripts/tests/yaml/runner.py
+++ b/scripts/tests/yaml/runner.py
@@ -256,7 +256,10 @@ def runner_base(ctx, configuration_directory: str, test_name: str, configuration
     tests_finder = TestsFinder(configuration_directory, configuration_name)
 
     parser_config = TestParserConfig(pics, specifications, kwargs)
-    parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger())
+    test_list = tests_finder.get(test_name)
+    if len(test_list) == 0:
+        raise Exception(f"No tests found for test name '{test_name}'")
+    parser_builder_config = TestParserBuilderConfig(test_list, parser_config, hooks=TestParserLogger())
     parser_builder_config.options.stop_on_error = stop_on_error
     while ctx:
         ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters)