Define CI tests arguments in YAML format (#35770)

* Define CI tests arguments in YAML format

* Convert ACE and ACL tests arguments to YAML

* Update doc for CI test arguments

* Restyled by prettier-markdown

* Improve error reporting

---------

Co-authored-by: Restyled.io <[email protected]>

docs/testing/python.md

@@ -644,74 +644,70 @@ single command.
 
 Example to compile all prerequisites and then running all python tests:
 
-```
+```shell
 ./scripts/tests/local.py build         # will compile python in out/pyenv and ALL application prerequisites
 ./scripts/tests/local.py python-tests  # Runs all python tests that are runnable in CI
 ```
 
 ## Defining the CI test arguments
 
-Below is the format of the structured environment definition comments:
+Arguments required to run a test can be defined in the comment block at the top
+of the test script. The section with the arguments should be placed between the
+`# === BEGIN CI TEST ARGUMENTS ===` and `# === END CI TEST ARGUMENTS ===`
+markers. Arguments should be structured as a valid YAML dictionary with a root
+key `test-runner-runs`, followed by the run identifier, and then the parameters
+for that run, e.g.:
 
-```
+```python
 # See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: <run_identifier>
-# test-runner-run/<run_identifier>/app: ${TYPE_OF_APP}
-# test-runner-run/<run_identifier>/factoryreset: <True|False>
-# test-runner-run/<run_identifier>/quiet: <True|False>
-# test-runner-run/<run_identifier>/app-args: <app_arguments>
-# test-runner-run/<run_identifier>/script-args: <script_arguments>
+# test-runner-runs:
+#   run1:
+#     app: ${TYPE_OF_APP}
+#     factoryreset: <true|false>
+#     quiet: <true|false>
+#     app-args: <app_arguments>
+#     script-args: <script_arguments>
 # === END CI TEST ARGUMENTS ===
 ```
 
-NOTE: The `=== BEGIN CI TEST ARGUMENTS ===` and `=== END CI TEST ARGUMENTS ===`
-markers must be present.
-
 ### Description of Parameters
 
--   `test-runner-runs`: Specifies the identifier for the run. This can be any
-    unique identifier.
-
-    -   Example: `run1`
-
--   `test-runner-run/<run_identifier>/app`: Indicates the application to be used
-    in the test. Different app types as needed could be referenced from section
-    [name: Generate an argument environment file ] of the file
+-   `app`: Indicates the application to be used in the test. Different app types
+    as needed could be referenced from section [name: Generate an argument
+    environment file ] of the file
     [.github/workflows/tests.yaml](https://github.com/project-chip/connectedhomeip/blob/master/.github/workflows/tests.yaml)
 
-        -   Example: `${TYPE_OF_APP}`
+    -   Example: `${TYPE_OF_APP}`
 
--   `test-runner-run/<run_identifier>/factoryreset`: Determines whether a
-    factory reset should be performed before the test.
+-   `factoryreset`: Determines whether a factory reset should be performed
+    before the test.
 
-    -   Example: `True`
+    -   Example: `true`
 
--   `test-runner-run/<run_identifier>/quiet`: Sets the verbosity level of the
-    test run. When set to True, the test run will be quieter.
+-   `quiet`: Sets the verbosity level of the test run. When set to True, the
+    test run will be quieter.
 
-    -   Example: `True`
+    -   Example: `true`
 
--   `test-runner-run/<run_identifier>/app-args`: Specifies the arguments to be
-    passed to the application during the test.
+-   `app-args`: Specifies the arguments to be passed to the application during
+    the test.
 
     -   Example:
         `--discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json`
 
--   `test-runner-run/<run_identifier>/script-args`: Specifies the arguments to
-    be passed to the test script.
+-   `script-args`: Specifies the arguments to be passed to the test script.
 
     -   Example:
         `--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto`
 
--   `test-runner-run/<run_identifier>/script-start-delay`: Specifies the number
-    of seconds to wait before starting the test script. This parameter can be
-    used to allow the application to initialize itself properly before the test
-    script will try to commission it (e.g. in case if the application needs to
-    be commissioned to some other controller first). By default, the delay is 0
-    seconds.
+-   `script-start-delay`: Specifies the number of seconds to wait before
+    starting the test script. This parameter can be used to allow the
+    application to initialize itself properly before the test script will try to
+    commission it (e.g. in case if the application needs to be commissioned to
+    some other controller first). By default, the delay is 0 seconds.
 
     -   Example: `10`
 

src/python_testing/TC_ACE_1_2.py

@@ -19,12 +19,19 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
-# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
-# test-runner-run/run1/factoryreset: True
-# test-runner-run/run1/quiet: True
-# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-runs:
+#   run1:
+#     app: ${ALL_CLUSTERS_APP}
+#     factoryreset: true
+#     quiet: true
+#     app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+#     script-args: >
+#       --storage-path admin_storage.json
+#       --commissioning-method on-network
+#       --discriminator 1234
+#       --passcode 20202021
+#       --trace-to json:${TRACE_TEST_JSON}.json
+#       --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import logging

src/python_testing/TC_ACE_1_3.py

@@ -19,12 +19,19 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
-# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
-# test-runner-run/run1/factoryreset: True
-# test-runner-run/run1/quiet: True
-# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-runs:
+#   run1:
+#     app: ${ALL_CLUSTERS_APP}
+#     factoryreset: true
+#     quiet: true
+#     app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+#     script-args: >
+#       --storage-path admin_storage.json
+#       --commissioning-method on-network
+#       --discriminator 1234
+#       --passcode 20202021
+#       --trace-to json:${TRACE_TEST_JSON}.json
+#       --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import logging

src/python_testing/TC_ACE_1_4.py

@@ -19,12 +19,21 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
-# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
-# test-runner-run/run1/factoryreset: True
-# test-runner-run/run1/quiet: True
-# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-runs:
+#   run1:
+#     app: ${ALL_CLUSTERS_APP}
+#     factoryreset: true
+#     quiet: true
+#     app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+#     script-args: >
+#       --storage-path admin_storage.json
+#       --commissioning-method on-network
+#       --discriminator 1234
+#       --passcode 20202021
+#       --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100
+#       --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff
+#       --trace-to json:${TRACE_TEST_JSON}.json
+#       --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import sys

src/python_testing/TC_ACE_1_5.py

@@ -19,12 +19,20 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
-# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
-# test-runner-run/run1/factoryreset: True
-# test-runner-run/run1/quiet: True
-# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-runs:
+#   run1:
+#     app: ${ALL_CLUSTERS_APP}
+#     factoryreset: true
+#     quiet: true
+#     app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+#     script-args: >
+#       --storage-path admin_storage.json
+#       --commissioning-method on-network
+#       --discriminator 1234
+#       --passcode 20202021
+#       --PICS src/app/tests/suites/certification/ci-pics-values
+#       --trace-to json:${TRACE_TEST_JSON}.json
+#       --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import logging

src/python_testing/TC_ACL_2_11.py

@@ -19,12 +19,23 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
-# test-runner-run/run1/app: ${NETWORK_MANAGEMENT_APP}
-# test-runner-run/run1/factoryreset: True
-# test-runner-run/run1/quiet: True
-# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json --commissioning-arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]" --arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]"
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-runs:
+#   run1:
+#     app: ${NETWORK_MANAGEMENT_APP}
+#     factoryreset: true
+#     quiet: true
+#     app-args: >
+#       --discriminator 1234 --KVS kvs1
+#       --trace-to json:${TRACE_APP}.json
+#       --commissioning-arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]"
+#       --arl-entries "[{\"endpoint\": 1,\"cluster\": 1105,\"restrictions\": [{\"type\": 0,\"id\": 0}]}]"
+#     script-args: >
+#       --storage-path admin_storage.json
+#       --commissioning-method on-network
+#       --discriminator 1234
+#       --passcode 20202021
+#       --trace-to json:${TRACE_TEST_JSON}.json
+#       --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import logging

src/python_testing/TC_ACL_2_2.py

@@ -16,12 +16,19 @@
 #
 
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
-# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
-# test-runner-run/run1/factoryreset: True
-# test-runner-run/run1/quiet: True
-# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-runs:
+#   run1:
+#     app: ${ALL_CLUSTERS_APP}
+#     factoryreset: true
+#     quiet: true
+#     app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+#     script-args: >
+#       --storage-path admin_storage.json
+#       --commissioning-method on-network
+#       --discriminator 1234
+#       --passcode 20202021
+#       --trace-to json:${TRACE_TEST_JSON}.json
+#       --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import chip.clusters as Clusters

src/python_testing/TC_AccessChecker.py

@@ -2,12 +2,19 @@
 # for details about the block below.
 #
 # === BEGIN CI TEST ARGUMENTS ===
-# test-runner-runs: run1
-# test-runner-run/run1/app: ${ALL_CLUSTERS_APP}
-# test-runner-run/run1/factoryreset: True
-# test-runner-run/run1/quiet: True
-# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
-# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
+# test-runner-runs:
+#   run1:
+#     app: ${ALL_CLUSTERS_APP}
+#     factoryreset: true
+#     quiet: true
+#     app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json
+#     script-args: >
+#       --storage-path admin_storage.json
+#       --commissioning-method on-network
+#       --discriminator 1234
+#       --passcode 20202021
+#       --trace-to json:${TRACE_TEST_JSON}.json
+#       --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto
 # === END CI TEST ARGUMENTS ===
 
 import logging