Skip to content

Commit

Permalink
Actually enable our YAML tests in the python-calls-chip-tool runner.
Browse files Browse the repository at this point in the history 
We were now getting the right test list, but for each test we were not finding
the test file.  And the harness did not treat that as an error.
  • Loading branch information
@bzbarsky-apple
bzbarsky-apple committed Sep 7, 2023
1 parent ba45f9c commit aa93df9
Show file tree
Hide file tree
Showing 4 changed files with 58 additions and 75 deletions.
16 changes: 12 additions & 4 deletions scripts/tests/chiptest/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,10 @@ def tests_with_command(chip_tool: str, is_manual: bool):
)


def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
"""
use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
"""
manual_tests = _GetManualTests()
flaky_tests = _GetFlakyTests()
slow_tests = _GetSlowTests()
Expand All @@ -270,21 +273,26 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool):
if treat_repl_unsupported_as_in_development and path.name in chip_repl_unsupported_tests:
tags.add(TestTag.IN_DEVELOPMENT)

if use_short_run_name:
run_name = path.stem # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
else:
run_name = str(path)

yield TestDefinition(
run_name=str(path),
run_name=run_name,
name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
target=target_for_name(path.name),
tags=tags,
)


def AllReplYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True):
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
yield test


def AllChipToolYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False):
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
yield test


Expand Down
5 changes: 4 additions & 1 deletion scripts/tests/yaml/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,10 @@ def runner_base(ctx, configuration_directory: str, test_name: str, configuration
tests_finder = TestsFinder(configuration_directory, configuration_name)

parser_config = TestParserConfig(pics, specifications, kwargs)
parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger())
test_list = tests_finder.get(test_name)
if len(test_list) == 0:
raise Exception(f"No tests found for test name '{test_name}'")
parser_builder_config = TestParserBuilderConfig(test_list, parser_config, hooks=TestParserLogger())
parser_builder_config.options.stop_on_error = stop_on_error
while ctx:
ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters)
Expand Down
94 changes: 34 additions & 60 deletions src/app/tests/suites/TestClusterMultiFabric.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,8 +284,7 @@ tests:
#
# TODO: This can be fixed using the `saveAs` function.
#
value:
[
value: [
{
FabricIndex: 1,
fabricSensitiveInt8u: 33,
Expand Down Expand Up @@ -328,39 +327,27 @@ tests:
},
{
FabricIndex: 2,
fabricSensitiveInt8u: 0,
# These should actually be missing, not null, but right
# now our harness treats those the same, and we have no
# way to indicate "missing" in the YAML.
fabricSensitiveInt8u: null,
optionalFabricSensitiveInt8u: null,
nullableFabricSensitiveInt8u: null,
fabricSensitiveCharString: "",
fabricSensitiveStruct:
{
a: 0,
b: false,
c: 0,
d: "",
e: "",
f: 0,
g: 0,
h: 0,
},
fabricSensitiveInt8uList: [],
fabricSensitiveCharString: null,
fabricSensitiveStruct: null,
fabricSensitiveInt8uList: null,
},
{
FabricIndex: 2,
fabricSensitiveInt8u: 0,
# These should actually be missing, not null, but right
# now our harness treats those the same, and we have no
# way to indicate "missing" in the YAML.
fabricSensitiveInt8u: null,
optionalFabricSensitiveInt8u: null,
nullableFabricSensitiveInt8u: null,
fabricSensitiveCharString: "",
fabricSensitiveStruct:
{
a: 0,
b: false,
c: 0,
d: "",
e: "",
f: 0,
g: 0,
h: 0,
},
fabricSensitiveInt8uList: [],
fabricSensitiveCharString: null,
fabricSensitiveStruct: null,
fabricSensitiveInt8uList: null,
},
]

Expand All @@ -371,43 +358,30 @@ tests:
fabricFiltered: false
attribute: "list_fabric_scoped"
response:
value:
[
value: [
{
FabricIndex: 1,
fabricSensitiveInt8u: 0,
# These should actually be missing, not null, but right
# now our harness treats those the same, and we have no
# way to indicate "missing" in the YAML.
fabricSensitiveInt8u: null,
optionalFabricSensitiveInt8u: null,
nullableFabricSensitiveInt8u: null,
fabricSensitiveCharString: "",
fabricSensitiveStruct:
{
a: 0,
b: false,
c: 0,
d: "",
e: "",
f: 0,
g: 0,
h: 0,
},
fabricSensitiveInt8uList: [],
fabricSensitiveCharString: null,
fabricSensitiveStruct: null,
fabricSensitiveInt8uList: null,
},
{
FabricIndex: 1,
fabricSensitiveInt8u: 0,
# These should actually be missing, not null, but right
# now our harness treats those the same, and we have no
# way to indicate "missing" in the YAML.
fabricSensitiveInt8u: null,
optionalFabricSensitiveInt8u: null,
nullableFabricSensitiveInt8u: null,
fabricSensitiveCharString: "",
fabricSensitiveStruct:
{
a: 0,
b: false,
c: 0,
d: "",
e: "",
f: 0,
g: 0,
h: 0,
},
fabricSensitiveInt8uList: [],
fabricSensitiveCharString: null,
fabricSensitiveStruct: null,
fabricSensitiveInt8uList: null,
},
{
FabricIndex: 2,
Expand Down
18 changes: 8 additions & 10 deletions src/app/tests/suites/TestEventsById.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ tests:
value: ReadRequestMessage.Cluster
- name: "EventId"
value: ReadRequestMessage.Event
response: []
response:
error: UNSUPPORTED_ENDPOINT

- label:
"Read Request Message with a path that indicates a specific cluster
Expand All @@ -62,11 +63,12 @@ tests:
value: UnsupportedCluster
- name: "EventId"
value: ReadRequestMessage.Event
response: []
response:
error: UNSUPPORTED_CLUSTER

- label:
"Read Request Message with a path that indicates a specific endpoint
that is unsupported"
"Read Request Message with a path that indicates a specific event that
is unsupported"
cluster: "AnyCommands"
command: "ReadEventById"
endpoint: ReadRequestMessage.EndPoint
Expand Down Expand Up @@ -256,9 +258,5 @@ tests:
- name: "EventId"
value: ReadRequestMessage.EndPoint
response:
- values:
- value: { arg1: 1, arg2: 2, arg3: true }
- values:
- value: { arg1: 3, arg2: 1, arg3: false }
- values:
- value: { arg1: 4, arg2: 3, arg3: true }
# Not a valid event request path.
error: FAILURE

0 comments on commit aa93df9

Please sign in to comment.