diff --git a/docs/testing/python.md b/docs/testing/python.md index c631550b040082..9c2edcfc85caf0 100644 --- a/docs/testing/python.md +++ b/docs/testing/python.md @@ -644,74 +644,70 @@ single command. Example to compile all prerequisites and then running all python tests: -``` +```shell ./scripts/tests/local.py build # will compile python in out/pyenv and ALL application prerequisites ./scripts/tests/local.py python-tests # Runs all python tests that are runnable in CI ``` ## Defining the CI test arguments -Below is the format of the structured environment definition comments: +Arguments required to run a test can be defined in the comment block at the top +of the test script. The section with the arguments should be placed between the +`# === BEGIN CI TEST ARGUMENTS ===` and `# === END CI TEST ARGUMENTS ===` +markers. Arguments should be structured as a valid YAML dictionary with a root +key `test-runner-runs`, followed by the run identifier, and then the parameters +for that run, e.g.: -``` +```python # See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments # for details about the block below. # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: -# test-runner-run//app: ${TYPE_OF_APP} -# test-runner-run//factoryreset: -# test-runner-run//quiet: -# test-runner-run//app-args: -# test-runner-run//script-args: +# test-runner-runs: +# run1: +# app: ${TYPE_OF_APP} +# factoryreset: +# quiet: +# app-args: +# script-args: # === END CI TEST ARGUMENTS === ``` -NOTE: The `=== BEGIN CI TEST ARGUMENTS ===` and `=== END CI TEST ARGUMENTS ===` -markers must be present. - ### Description of Parameters -- `test-runner-runs`: Specifies the identifier for the run. This can be any - unique identifier. - - - Example: `run1` - -- `test-runner-run//app`: Indicates the application to be used - in the test. Different app types as needed could be referenced from section - [name: Generate an argument environment file ] of the file +- `app`: Indicates the application to be used in the test. Different app types + as needed could be referenced from section [name: Generate an argument + environment file ] of the file [.github/workflows/tests.yaml](https://github.com/project-chip/connectedhomeip/blob/master/.github/workflows/tests.yaml) - - Example: `${TYPE_OF_APP}` + - Example: `${TYPE_OF_APP}` -- `test-runner-run//factoryreset`: Determines whether a - factory reset should be performed before the test. +- `factoryreset`: Determines whether a factory reset should be performed + before the test. - - Example: `True` + - Example: `true` -- `test-runner-run//quiet`: Sets the verbosity level of the - test run. When set to True, the test run will be quieter. +- `quiet`: Sets the verbosity level of the test run. When set to True, the + test run will be quieter. - - Example: `True` + - Example: `true` -- `test-runner-run//app-args`: Specifies the arguments to be - passed to the application during the test. +- `app-args`: Specifies the arguments to be passed to the application during + the test. - Example: `--discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json` -- `test-runner-run//script-args`: Specifies the arguments to - be passed to the test script. +- `script-args`: Specifies the arguments to be passed to the test script. - Example: `--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto` -- `test-runner-run//script-start-delay`: Specifies the number - of seconds to wait before starting the test script. This parameter can be - used to allow the application to initialize itself properly before the test - script will try to commission it (e.g. in case if the application needs to - be commissioned to some other controller first). By default, the delay is 0 - seconds. +- `script-start-delay`: Specifies the number of seconds to wait before + starting the test script. This parameter can be used to allow the + application to initialize itself properly before the test script will try to + commission it (e.g. in case if the application needs to be commissioned to + some other controller first). By default, the delay is 0 seconds. - Example: `10` diff --git a/src/python_testing/TC_ACE_1_2.py b/src/python_testing/TC_ACE_1_2.py index 9bd26523f3cbba..7d227f5a5c0e72 100644 --- a/src/python_testing/TC_ACE_1_2.py +++ b/src/python_testing/TC_ACE_1_2.py @@ -19,12 +19,19 @@ # for details about the block below. # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: run1 -# test-runner-run/run1/app: ${ALL_CLUSTERS_APP} -# test-runner-run/run1/factoryreset: True -# test-runner-run/run1/quiet: True -# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json -# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# test-runner-runs: +# run1: +# app: ${ALL_CLUSTERS_APP} +# factoryreset: true +# quiet: true +# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# script-args: > +# --storage-path admin_storage.json +# --commissioning-method on-network +# --discriminator 1234 +# --passcode 20202021 +# --trace-to json:${TRACE_TEST_JSON}.json +# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # === END CI TEST ARGUMENTS === import logging diff --git a/src/python_testing/TC_ACE_1_3.py b/src/python_testing/TC_ACE_1_3.py index 3c64171571eb95..29aabf3462083b 100644 --- a/src/python_testing/TC_ACE_1_3.py +++ b/src/python_testing/TC_ACE_1_3.py @@ -19,12 +19,19 @@ # for details about the block below. # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: run1 -# test-runner-run/run1/app: ${ALL_CLUSTERS_APP} -# test-runner-run/run1/factoryreset: True -# test-runner-run/run1/quiet: True -# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json -# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# test-runner-runs: +# run1: +# app: ${ALL_CLUSTERS_APP} +# factoryreset: true +# quiet: true +# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# script-args: > +# --storage-path admin_storage.json +# --commissioning-method on-network +# --discriminator 1234 +# --passcode 20202021 +# --trace-to json:${TRACE_TEST_JSON}.json +# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # === END CI TEST ARGUMENTS === import logging diff --git a/src/python_testing/TC_ACE_1_4.py b/src/python_testing/TC_ACE_1_4.py index f6f04e3508860b..9344b0bd3e9876 100644 --- a/src/python_testing/TC_ACE_1_4.py +++ b/src/python_testing/TC_ACE_1_4.py @@ -19,12 +19,21 @@ # for details about the block below. # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: run1 -# test-runner-run/run1/app: ${ALL_CLUSTERS_APP} -# test-runner-run/run1/factoryreset: True -# test-runner-run/run1/quiet: True -# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json -# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# test-runner-runs: +# run1: +# app: ${ALL_CLUSTERS_APP} +# factoryreset: true +# quiet: true +# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# script-args: > +# --storage-path admin_storage.json +# --commissioning-method on-network +# --discriminator 1234 +# --passcode 20202021 +# --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100 +# --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff +# --trace-to json:${TRACE_TEST_JSON}.json +# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # === END CI TEST ARGUMENTS === import sys diff --git a/src/python_testing/TC_ACE_1_5.py b/src/python_testing/TC_ACE_1_5.py index 00bd343d461cb3..cda418737af048 100644 --- a/src/python_testing/TC_ACE_1_5.py +++ b/src/python_testing/TC_ACE_1_5.py @@ -19,12 +19,20 @@ # for details about the block below. # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: run1 -# test-runner-run/run1/app: ${ALL_CLUSTERS_APP} -# test-runner-run/run1/factoryreset: True -# test-runner-run/run1/quiet: True -# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json -# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# test-runner-runs: +# run1: +# app: ${ALL_CLUSTERS_APP} +# factoryreset: true +# quiet: true +# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# script-args: > +# --storage-path admin_storage.json +# --commissioning-method on-network +# --discriminator 1234 +# --passcode 20202021 +# --PICS src/app/tests/suites/certification/ci-pics-values +# --trace-to json:${TRACE_TEST_JSON}.json +# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # === END CI TEST ARGUMENTS === import logging diff --git a/src/python_testing/TC_ACL_2_11.py b/src/python_testing/TC_ACL_2_11.py index f5ebed221b272b..88ef1d26cee1d1 100644 --- a/src/python_testing/TC_ACL_2_11.py +++ b/src/python_testing/TC_ACL_2_11.py @@ -19,12 +19,23 @@ # for details about the block below. # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: run1 -# test-runner-run/run1/app: ${NETWORK_MANAGEMENT_APP} -# test-runner-run/run1/factoryreset: True -# test-runner-run/run1/quiet: True -# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json --commissioning-arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]" --arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]" -# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# test-runner-runs: +# run1: +# app: ${NETWORK_MANAGEMENT_APP} +# factoryreset: true +# quiet: true +# app-args: > +# --discriminator 1234 --KVS kvs1 +# --trace-to json:${TRACE_APP}.json +# --commissioning-arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]" +# --arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]" +# script-args: > +# --storage-path admin_storage.json +# --commissioning-method on-network +# --discriminator 1234 +# --passcode 20202021 +# --trace-to json:${TRACE_TEST_JSON}.json +# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # === END CI TEST ARGUMENTS === import logging diff --git a/src/python_testing/TC_ACL_2_2.py b/src/python_testing/TC_ACL_2_2.py index a1e755b7397f88..b0880b096bdb5a 100644 --- a/src/python_testing/TC_ACL_2_2.py +++ b/src/python_testing/TC_ACL_2_2.py @@ -16,12 +16,19 @@ # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: run1 -# test-runner-run/run1/app: ${ALL_CLUSTERS_APP} -# test-runner-run/run1/factoryreset: True -# test-runner-run/run1/quiet: True -# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json -# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# test-runner-runs: +# run1: +# app: ${ALL_CLUSTERS_APP} +# factoryreset: true +# quiet: true +# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# script-args: > +# --storage-path admin_storage.json +# --commissioning-method on-network +# --discriminator 1234 +# --passcode 20202021 +# --trace-to json:${TRACE_TEST_JSON}.json +# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # === END CI TEST ARGUMENTS === import chip.clusters as Clusters diff --git a/src/python_testing/TC_AccessChecker.py b/src/python_testing/TC_AccessChecker.py index 36c643407937e2..a2429ee3540147 100644 --- a/src/python_testing/TC_AccessChecker.py +++ b/src/python_testing/TC_AccessChecker.py @@ -2,12 +2,19 @@ # for details about the block below. # # === BEGIN CI TEST ARGUMENTS === -# test-runner-runs: run1 -# test-runner-run/run1/app: ${ALL_CLUSTERS_APP} -# test-runner-run/run1/factoryreset: True -# test-runner-run/run1/quiet: True -# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json -# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# test-runner-runs: +# run1: +# app: ${ALL_CLUSTERS_APP} +# factoryreset: true +# quiet: true +# app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# script-args: > +# --storage-path admin_storage.json +# --commissioning-method on-network +# --discriminator 1234 +# --passcode 20202021 +# --trace-to json:${TRACE_TEST_JSON}.json +# --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto # === END CI TEST ARGUMENTS === import logging diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py b/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py index aa829c40d855c7..803160e5c736e5 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/metadata.py @@ -12,17 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import re -import sys from dataclasses import dataclass +from io import StringIO from typing import Any, Dict, List import yaml -def bool_from_str(value: str) -> bool: +# TODO #35787: Remove support for non-YAML format +def cast_to_bool(value: Any) -> bool: """Convert True/true/False/false strings to bool.""" - return value.strip().lower() == "true" + if isinstance(value, str): + return value.strip().lower() == "true" + return bool(value) @dataclass @@ -67,16 +71,22 @@ def copy_from_dict(self, attr_dict: Dict[str, Any]) -> None: self.py_script_path = attr_dict["py_script_path"] if "factoryreset" in attr_dict: - self.factoryreset = bool_from_str(attr_dict["factoryreset"]) + self.factoryreset = cast_to_bool(attr_dict["factoryreset"]) if "factoryreset_app_only" in attr_dict: - self.factoryreset_app_only = bool_from_str(attr_dict["factoryreset_app_only"]) + self.factoryreset_app_only = cast_to_bool(attr_dict["factoryreset_app_only"]) if "script_gdb" in attr_dict: - self.script_gdb = bool_from_str(attr_dict["script_gdb"]) + self.script_gdb = cast_to_bool(attr_dict["script_gdb"]) if "quiet" in attr_dict: - self.quiet = bool_from_str(attr_dict["quiet"]) + self.quiet = cast_to_bool(attr_dict["quiet"]) + + +class NamedStringIO(StringIO): + def __init__(self, content, name): + super().__init__(content) + self.name = name def extract_runs_arg_lines(py_script_path: str) -> Dict[str, Dict[str, str]]: @@ -91,23 +101,34 @@ def extract_runs_arg_lines(py_script_path: str) -> Dict[str, Dict[str, str]]: runs_arg_lines: Dict[str, Dict[str, str]] = {} + ci_args_section_lines = [] with open(py_script_path, 'r', encoding='utf8') as py_script: for line_idx, line in enumerate(py_script.readlines()): line = line.strip() line_num = line_idx + 1 + # Append empty line to the line capture, so during YAML parsing + # line numbers will match the original file. + ci_args_section_lines.append("") + # Detect the single CI args section, to skip the lines otherwise. if not done_ci_args_section and line.startswith("# === BEGIN CI TEST ARGUMENTS ==="): found_ci_args_section = True + continue elif found_ci_args_section and line.startswith("# === END CI TEST ARGUMENTS ==="): done_ci_args_section = True found_ci_args_section = False + continue + + if found_ci_args_section: + # Update the last line in the line capture. + ci_args_section_lines[-1] = " " + line.lstrip("#") runs_match = runs_def_ptrn.match(line) args_match = arg_def_ptrn.match(line) if not found_ci_args_section and (runs_match or args_match): - print(f"WARNING: {py_script_path}:{line_num}: Found CI args outside of CI TEST ARGUMENTS block!", file=sys.stderr) + logging.warning(f"{py_script_path}:{line_num}: Found CI args outside of CI TEST ARGUMENTS block") continue if runs_match: @@ -119,6 +140,17 @@ def extract_runs_arg_lines(py_script_path: str) -> Dict[str, Dict[str, str]]: elif args_match: runs_arg_lines[args_match.group("run_id")][args_match.group("arg_name")] = args_match.group("arg_val") + if not runs_arg_lines: + try: + runs = yaml.safe_load(NamedStringIO("\n".join(ci_args_section_lines), py_script_path)) + for run, args in runs.get("test-runner-runs", {}).items(): + runs_arg_lines[run] = {} + runs_arg_lines[run]['run'] = run + runs_arg_lines[run]['py_script_path'] = py_script_path + runs_arg_lines[run].update(args) + except yaml.YAMLError as e: + logging.error(f"Failed to parse CI arguments YAML: {e}") + return runs_arg_lines @@ -155,10 +187,12 @@ def __resolve_env_vals__(self, metadata_dict: Dict[str, str]) -> None: the value for that argument defined in the test script. """ for arg, arg_val in metadata_dict.items(): + if not isinstance(arg_val, str): + continue # We do not expect to recurse (like ${FOO_${BAR}}) so just expand once for name, value in self.env.items(): arg_val = arg_val.replace(f'${{{name}}}', value) - metadata_dict[arg] = arg_val + metadata_dict[arg] = arg_val.strip() def parse_script(self, py_script_path: str) -> List[Metadata]: """ diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/test_metadata.py b/src/python_testing/matter_testing_infrastructure/chip/testing/test_metadata.py index 863add965c315e..ce41969fe0af1f 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/test_metadata.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/test_metadata.py @@ -34,6 +34,21 @@ class TestMetadataReader(unittest.TestCase): # test-runner-run/run1/quiet: False ''' + test_file_content_yaml = ''' + # === BEGIN CI TEST ARGUMENTS === + # test-runner-runs: + # run1: + # app: ${ALL_CLUSTERS_APP} + # app-args: --discriminator 1234 --trace-to json:${TRACE_APP}.json + # script-args: > + # --commissioning-method on-network + # --trace-to json:${TRACE_TEST_JSON}.json + # --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto + # factoryreset: true + # quiet: true + # === END CI TEST ARGUMENTS === + ''' + env_file_content = ''' ALL_CLUSTERS_APP: out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app CHIP_LOCK_APP: out/linux-x64-lock-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-lock-app @@ -61,15 +76,21 @@ def generate_temp_file(self, directory: str, file_content: str) -> str: def test_run_arg_generation(self): with tempfile.TemporaryDirectory() as temp_dir: - temp_file = self.generate_temp_file(temp_dir, self.test_file_content) + test_file = self.generate_temp_file(temp_dir, self.test_file_content) env_file = self.generate_temp_file(temp_dir, self.env_file_content) reader = MetadataReader(env_file) - self.maxDiff = None + self.expected_metadata.py_script_path = test_file + self.assertEqual(self.expected_metadata, reader.parse_script(test_file)[0]) - self.expected_metadata.py_script_path = temp_file - actual = reader.parse_script(temp_file)[0] - self.assertEqual(self.expected_metadata, actual) + def test_run_arg_generation_yaml(self): + with tempfile.TemporaryDirectory() as temp_dir: + test_file = self.generate_temp_file(temp_dir, self.test_file_content_yaml) + env_file = self.generate_temp_file(temp_dir, self.env_file_content) + + reader = MetadataReader(env_file) + self.expected_metadata.py_script_path = test_file + self.assertEqual(self.expected_metadata, reader.parse_script(test_file)[0]) if __name__ == "__main__":