From 95e5647a5bcf0889aa464b940be2fc10215d66c1 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Tue, 2 May 2023 16:26:33 +0200 Subject: [PATCH] [chiptool.py] Add --trace_file, --trace_decode, --node_id, --continueOnFailure, --timeout, --delayInMs supports --- .../matter_yamltests/parser.py | 13 ++++++-- .../matter_yamltests/runner.py | 11 ++++++- scripts/tests/yaml/chiptool.py | 33 ++++++++++++++++++- scripts/tests/yaml/runner.py | 14 ++++---- 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser.py b/scripts/py_matter_yamltests/matter_yamltests/parser.py index 1affdf21fd6564..33ace02d5aca8c 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/parser.py +++ b/scripts/py_matter_yamltests/matter_yamltests/parser.py @@ -962,8 +962,8 @@ def __init__(self, test_file: str, parser_config: TestParserConfig = TestParserC yaml_loader = YamlLoader() filename, name, pics, config, tests = yaml_loader.load(test_file) - self.__apply_config_override(config, parser_config.config_override) self.__apply_legacy_config(config) + self.__apply_config_override(config, parser_config.config_override) self.filename = filename self.name = name @@ -974,12 +974,21 @@ def __init__(self, test_file: str, parser_config: TestParserConfig = TestParserC PICSChecker(parser_config.pics), tests ) + self.timeout = config['timeout'] def __apply_config_override(self, config, config_override): for key, value in config_override.items(): - if value is None: + if value is None or not key in config: continue + if type(value) is str: + if key == 'timeout' or key == 'endpoint': + value = int(value) + elif key == 'nodeId' and value.startswith('0x'): + value = int(value, 16) + elif key == 'nodeId': + value = int(value) + if isinstance(config[key], dict) and 'defaultValue' in config[key]: config[key]['defaultValue'] = value else: diff --git a/scripts/py_matter_yamltests/matter_yamltests/runner.py b/scripts/py_matter_yamltests/matter_yamltests/runner.py index 884d2c249467df..3229dace1301bf 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/runner.py +++ b/scripts/py_matter_yamltests/matter_yamltests/runner.py @@ -43,10 +43,14 @@ class TestRunnerOptions: stop running the tests if a step index matches the number. This is mostly useful when running a single test file and for debugging purposes. + + delay_in_ms: If set to any value that is not zero the runner will + wait for the given time between steps. """ stop_on_error: bool = True stop_on_warning: bool = False stop_at_number: int = -1 + delay_in_ms: int = 0 @dataclass @@ -139,7 +143,9 @@ def run(self, parser_builder_config: TestParserBuilderConfig, runner_config: Tes if not parser or not runner_config: continue - result = asyncio.run(self._run(parser, runner_config)) + loop = asyncio.get_event_loop() + result = loop.run_until_complete(asyncio.wait_for( + self._run(parser, runner_config), parser.timeout)) if isinstance(result, Exception): raise (result) elif not result: @@ -200,6 +206,9 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig): if (idx + 1) == config.options.stop_at_number: break + if config.options.delay_in_ms: + await asyncio.sleep(config.options.delay_in_ms / 1000) + hooks.test_stop(round(test_duration)) except Exception as exception: diff --git a/scripts/tests/yaml/chiptool.py b/scripts/tests/yaml/chiptool.py index d47a4d22d32882..81884b82687253 100755 --- a/scripts/tests/yaml/chiptool.py +++ b/scripts/tests/yaml/chiptool.py @@ -79,6 +79,14 @@ def chiptool_runner_options(f): help='Name of a websocket server to run at launch.')(f) f = click.option('--server_arguments', type=str, default='interactive server', help='Optional arguments to pass to the websocket server at launch.')(f) + f = click.option('--trace_file', type=click.Path(), default=None, + help='Optional file path to save the tracing output to.')(f) + f = click.option('--trace_decode', type=bool, default=True, + help='Decode the tracing ouput to a human readable format.')(f) + f = click.option('--delay-in-ms', '--delayInMs', type=int, default=0, show_default=True, + help='Add a delay between each test suite steps.')(f) + f = click.option('--continueOnFailure', type=bool, default=False, show_default=True, + help='Do not stop running the test suite on first error.')(f) f = click.option('--PICS', type=click.Path(exists=True), show_default=True, default=_DEFAULT_PICS_FILE, help='Path to the PICS file to use.')(f) return f @@ -88,13 +96,36 @@ def chiptool_runner_options(f): CONTEXT_SETTINGS['default_map']['chiptool']['use_test_harness_log_format'] = True +def maybe_update_server_arguments(ctx): + if ctx.params['trace_file']: + ctx.params['server_arguments'] += ' --trace_file {}'.format(ctx.params['trace_file']) + + if ctx.params['trace_decode']: + ctx.params['server_arguments'] += ' --trace_decode 1' + + del ctx.params['trace_file'] + del ctx.params['trace_decode'] + + return ctx.params['server_arguments'] + + +def maybe_update_stop_on_error(ctx): + if ctx.params['continueonfailure']: + ctx.params['stop_on_error'] = False + + del ctx.params['continueonfailure'] + + @click.command(context_settings=CONTEXT_SETTINGS) @click.argument('commands', nargs=-1) @chiptool_runner_options @click.pass_context -def chiptool_py(ctx, commands: list[str], server_path: str, server_name: str, server_arguments: str, pics: str): +def chiptool_py(ctx, commands: list[str], server_path: str, server_name: str, server_arguments: str, trace_file: str, trace_decode: bool, delay_in_ms: int, continueonfailure: bool, pics: str): success = False + server_arguments = maybe_update_server_arguments(ctx) + maybe_update_stop_on_error(ctx) + if len(commands) > 1 and commands[0] == 'tests': success = send_yaml_command(commands[1], server_path, server_arguments, pics, commands[2:]) else: diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/yaml/runner.py index 7ccc6cf50253ea..4b60d5d1ab25b1 100755 --- a/scripts/tests/yaml/runner.py +++ b/scripts/tests/yaml/runner.py @@ -73,6 +73,8 @@ def test_runner_options(f): help='Show additional logs provided by the adapter on error.')(f) f = click.option('--use_test_harness_log_format', type=bool, default=False, show_default=True, help='Use the test harness log format.')(f) + f = click.option('--delay-in-ms', type=int, default=0, show_default=True, + help='Add a delay between test suite steps.')(f) return f @@ -263,10 +265,10 @@ def dry_run(parser_group: ParserGroup): @runner_base.command() @test_runner_options @pass_parser_group -def run(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, use_test_harness_log_format: bool): +def run(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, use_test_harness_log_format: bool, delay_in_ms: int): """Run the test suite.""" adapter = __import__(adapter, fromlist=[None]).Adapter(parser_group.builder_config.parser_config.definitions) - runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number) + runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number, delay_in_ms) runner_hooks = TestRunnerLogger(show_adapter_logs, show_adapter_logs_on_error, use_test_harness_log_format) runner_config = TestRunnerConfig(adapter, parser_group.pseudo_clusters, runner_options, runner_hooks) @@ -278,10 +280,10 @@ def run(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_wa @test_runner_options @websocket_runner_options @pass_parser_group -def websocket(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, use_test_harness_log_format: bool, server_address: str, server_port: int, server_path: str, server_name: str, server_arguments: str): +def websocket(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, use_test_harness_log_format: bool, delay_in_ms: int, server_address: str, server_port: int, server_path: str, server_name: str, server_arguments: str): """Run the test suite using websockets.""" adapter = __import__(adapter, fromlist=[None]).Adapter(parser_group.builder_config.parser_config.definitions) - runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number) + runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number, delay_in_ms) runner_hooks = TestRunnerLogger(show_adapter_logs, show_adapter_logs_on_error, use_test_harness_log_format) runner_config = TestRunnerConfig(adapter, parser_group.pseudo_clusters, runner_options, runner_hooks) @@ -301,10 +303,10 @@ def websocket(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop @test_runner_options @chip_repl_runner_options @pass_parser_group -def chip_repl(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, use_test_harness_log_format: bool, runner: str, repl_storage_path: str, commission_on_network_dut: bool): +def chip_repl(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, use_test_harness_log_format: bool, delay_in_ms: int, runner: str, repl_storage_path: str, commission_on_network_dut: bool): """Run the test suite using chip-repl.""" adapter = __import__(adapter, fromlist=[None]).Adapter(parser_group.builder_config.parser_config.definitions) - runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number) + runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number, delay_in_ms) runner_hooks = TestRunnerLogger(show_adapter_logs, show_adapter_logs_on_error, use_test_harness_log_format) runner_config = TestRunnerConfig(adapter, parser_group.pseudo_clusters, runner_options, runner_hooks)