Skip to content

Commit

Permalink
[MatterYamlTests][darwin-framework-tool] Use a list of tests to run f…
Browse files Browse the repository at this point in the history 
…rom scripts/tests/chiptest/__init__.py if chip_tool_python is used in conjunction with darwinframeworktool.py
  • Loading branch information
@vivien-apple
vivien-apple committed Feb 2, 2024
1 parent 51392b0 commit ec57220
Show file tree
Hide file tree
Showing 4 changed files with 105 additions and 48 deletions.
1 change: 1 addition & 0 deletions .github/workflows/darwin-tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ jobs:
run: |
./scripts/run_in_build_env.sh \
"./scripts/tests/run_test_suite.py \
--runner darwin_framework_tool_python \
--chip-tool ./out/darwin-x64-darwin-framework-tool-${BUILD_VARIANT_FRAMEWORK_TOOL}/darwin-framework-tool \
--target-skip-glob '{TestAccessControlConstraints}' \
run \
Expand Down
66 changes: 63 additions & 3 deletions scripts/tests/chiptest/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,57 @@ def _GetInDevelopmentTests() -> Set[str]:
}


def _GetDarwinFrameworkToolUnsupportedTests() -> Set[str]:
"""Tests that fail in darwin-framework-tool for some reason"""
return {
"DL_LockUnlock", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_AddNewFabricFromExistingFabric", # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
"TestAttributesById", # The name of the arguments once converted differs for chip-tool and darwin-framework-tool
"TestBasicInformation", # darwin-framework-tool does not support writing readonly attributes by name
"TestClusterComplexTypes", # Test Failure : The test expectation "wasPresent (0) == True" is false
"TestClusterMultiFabric", # When reading TestFabricScoped, the result differs
"TestCommandsById", # darwin-framework-tool does not support writing readonly attributes by name
"TestDiagnosticLogs", # darwin-framework-tool does not implement a BDXTransferServerDelegate
"TestDiscovery", # darwin-framework-tool does not support dns-sd commands.
"TestEvents", # darwin-framework-tool does not currently support reading or subscribing to Events
"TestEventsById", # darwin-framework-tool does not currently support reading or subscribing to Events
"TestGroupMessaging", # darwin-framework-tool does not support group commands.
"TestIcdManagementCluster", # darwin-framework-tool does not support ICD registration
"TestUnitTestingClusterMei", # darwin-framework-tool does not currently support reading or subscribing to Events
"TestReadNoneSubscribeNone", # darwin-framework-tool does not supports those commands.

"Test_TC_ACE_1_6", # darwin-framework-tool does not support group commands.
"Test_TC_ACL_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_7", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_8", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_9", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_10", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_BINFO_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_BINFO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_BRBINFO_2_1", # Test Failure : The test expects the "UNSUPPORTED_ATTRIBUTE" error but the "FAILURE" error occured.
"Test_TC_DGGEN_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_DRLK_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_DGTHREAD_2_1",
"Test_TC_DGTHREAD_2_2",
"Test_TC_DGTHREAD_2_3",
"Test_TC_DGTHREAD_2_4",
"Test_TC_FLABEL_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_GRPKEY_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_LCFG_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_OPCREDS_3_7", # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
"Test_TC_OPSTATE_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SC_4_1", # darwin-framework-tool does not support dns-sd commands.
"Test_TC_SC_5_2", # darwin-framework-tool does not support group commands.
"Test_TC_S_2_3", # darwin-framework-tool does not support group commands.
}


def _GetChipReplUnsupportedTests() -> Set[str]:
"""Tests that fail in chip-repl for some reason"""
return {
Expand Down Expand Up @@ -284,7 +335,7 @@ def tests_with_command(chip_tool: str, is_manual: bool):
)


def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, treat_dft_unsupported_as_in_development: bool, use_short_run_name: bool):
"""
use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
"""
Expand All @@ -294,6 +345,7 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short
extra_slow_tests = _GetExtraSlowTests()
in_development_tests = _GetInDevelopmentTests()
chip_repl_unsupported_tests = _GetChipReplUnsupportedTests()
treat_dft_unsupported_as_in_development_tests = _GetDarwinFrameworkToolUnsupportedTests()
purposeful_failure_tests = _GetPurposefulFailureTests()

for path in _AllYamlTests():
Expand Down Expand Up @@ -327,6 +379,9 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short
else:
run_name = str(path)

if treat_dft_unsupported_as_in_development and run_name in treat_dft_unsupported_as_in_development_tests:
tags.add(TestTag.IN_DEVELOPMENT)

yield TestDefinition(
run_name=run_name,
name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
Expand All @@ -336,12 +391,17 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short


def AllReplYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, treat_dft_unsupported_as_in_development=False, use_short_run_name=False):
yield test


def AllChipToolYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, use_short_run_name=True):
yield test


def AllDarwinFrameworkToolYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=True, use_short_run_name=True):
yield test


Expand Down
58 changes: 27 additions & 31 deletions scripts/tests/chiptest/test_definition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,10 +251,9 @@ def to_s(self):


class TestRunTime(Enum):
CHIP_TOOL_BUILTIN = auto() # run via chip-tool built-in test commands
CHIP_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool
DARWIN_FRAMEWORK_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool
CHIP_REPL_PYTHON = auto() # use the python yaml test runner
DARWIN_FRAMEWORK_TOOL_BUILTIN = auto() # run via darwin-framework-tool built-in test commands


@dataclass
Expand All @@ -281,7 +280,7 @@ def tags_str(self) -> str:
return ", ".join([t.to_s() for t in self.tags])

def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_BUILTIN):
timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_PYTHON):
"""
Executes the given test case using the provided runner for execution.
"""
Expand Down Expand Up @@ -333,8 +332,6 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
# so it will be commissionable again.
app.factoryReset()

tool_cmd = paths.chip_tool if test_runtime != TestRunTime.CHIP_TOOL_PYTHON else paths.chip_tool_with_python_cmd

if dry_run:
tool_storage_dir = None
tool_storage_args = []
Expand All @@ -350,38 +347,37 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
app.start()
setupCode = app.setupCode

pairing_cmd = tool_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
test_cmd = tool_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
if test_runtime == TestRunTime.CHIP_TOOL_PYTHON:
if test_runtime == TestRunTime.CHIP_REPL_PYTHON:
chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
python_cmd = chip_repl_yaml_tester_cmd + \
['--setup-code', setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
if dry_run:
logging.info(" ".join(python_cmd))
else:
runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
dependencies=[apps_register], timeout_seconds=timeout_seconds)
else:
pairing_cmd = paths.chip_tool_with_python_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
test_cmd = paths.chip_tool_with_python_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
server_args = ['--server_path', paths.chip_tool[-1]] + \
['--server_arguments', 'interactive server' +
(' ' if len(tool_storage_args) else '') + ' '.join(tool_storage_args)]
pairing_cmd += server_args
test_cmd += server_args
elif test_runtime == TestRunTime.CHIP_TOOL_BUILTIN:
pairing_cmd += tool_storage_args
test_cmd += tool_storage_args

if dry_run:
# Some of our command arguments have spaces in them, so if we are
# trying to log commands people can run we should quote those.
def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
logging.info(" ".join(map(quoter, pairing_cmd)))
logging.info(" ".join(map(quoter, test_cmd)))
elif test_runtime == TestRunTime.CHIP_REPL_PYTHON:
chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
python_cmd = chip_repl_yaml_tester_cmd + \
['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
dependencies=[apps_register], timeout_seconds=timeout_seconds)
else:
runner.RunSubprocess(pairing_cmd,
name='PAIR', dependencies=[apps_register])

runner.RunSubprocess(
test_cmd,
name='TEST', dependencies=[apps_register],
timeout_seconds=timeout_seconds)
if dry_run:
# Some of our command arguments have spaces in them, so if we are
# trying to log commands people can run we should quote those.
def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
logging.info(" ".join(map(quoter, pairing_cmd)))
logging.info(" ".join(map(quoter, test_cmd)))
else:
runner.RunSubprocess(pairing_cmd,
name='PAIR', dependencies=[apps_register])
runner.RunSubprocess(
test_cmd,
name='TEST', dependencies=[apps_register],
timeout_seconds=timeout_seconds)

except Exception:
logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!")
Expand Down
28 changes: 14 additions & 14 deletions scripts/tests/run_test_suite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ class RunContext:
)
@click.option(
'--runner',
type=click.Choice(['codegen', 'chip_repl_python', 'chip_tool_python'], case_sensitive=False),
default='codegen',
type=click.Choice(['chip_repl_python', 'chip_tool_python', 'darwin_framework_tool_python'], case_sensitive=False),
default='chip_tool_python',
help='Run YAML tests using the specified runner.')
@click.option(
'--chip-tool',
Expand All @@ -138,18 +138,18 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
log_fmt = '%(levelname)-7s %(message)s'
coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)

runtime = TestRunTime.CHIP_TOOL_BUILTIN
runtime = TestRunTime.CHIP_TOOL_PYTHON
if runner == 'chip_repl_python':
runtime = TestRunTime.CHIP_REPL_PYTHON
elif runner == 'chip_tool_python':
runtime = TestRunTime.CHIP_TOOL_PYTHON
elif chip_tool is not None and os.path.basename(chip_tool) == "darwin-framework-tool":
runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_BUILTIN
elif runner == 'darwin_framework_tool_python':
runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON

if chip_tool is None and not runtime == TestRunTime.CHIP_REPL_PYTHON:
# non yaml tests REQUIRE chip-tool. Yaml tests should not require chip-tool
paths_finder = PathsFinder()
chip_tool = paths_finder.get('chip-tool')
if runtime == TestRunTime.CHIP_TOOL_PYTHON:
chip_tool = paths_finder.get('chip-tool')
else:
chip_tool = paths_finder.get('darwin-framework-tool')

if include_tags:
include_tags = set([TestTag.__members__[t] for t in include_tags])
Expand All @@ -160,10 +160,10 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
# Figures out selected test that match the given name(s)
if runtime == TestRunTime.CHIP_REPL_PYTHON:
all_tests = [test for test in chiptest.AllReplYamlTests()]
elif runtime == TestRunTime.CHIP_TOOL_PYTHON and os.path.basename(chip_tool) != "darwin-framework-tool":
all_tests = [test for test in chiptest.AllChipToolYamlTests()]
elif runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
all_tests = [test for test in chiptest.AllDarwinFrameworkToolYamlTests()]
else:
all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)]
all_tests = [test for test in chiptest.AllChipToolYamlTests()]

tests = all_tests

Expand All @@ -178,7 +178,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
TestTag.PURPOSEFUL_FAILURE,
}

if runtime != TestRunTime.CHIP_TOOL_PYTHON:
if runtime == TestRunTime.CHIP_REPL_PYTHON:
exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)

if 'all' not in target:
Expand Down Expand Up @@ -319,7 +319,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o
chip_repl_yaml_tester = paths_finder.get('yamltest_with_chip_repl_tester.py')

if chip_tool_with_python is None:
if context.obj.chip_tool and os.path.basename(context.obj.chip_tool) == "darwin-framework-tool":
if context.obj.runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
chip_tool_with_python = paths_finder.get('darwinframeworktool.py')
else:
chip_tool_with_python = paths_finder.get('chiptool.py')
Expand Down

0 comments on commit ec57220

Please sign in to comment.