Skip to content

Commit

Permalink
Re-enable TC-RR-1.1 in CI
Browse files Browse the repository at this point in the history
- A regression had occured on RR-1.1 in master before CI was attempted
  in project-chip#22143 which prevented the CI test to be enabled

This PR:

- Fixes TC-RR-1.1
- Enables TC-RR-1.1, TC-SC-3.6 and TC-DA-1.7 to keep them fresh and
  improve integration test coverage (TC-RR-1.1 tests a lot)
- Improve run_python_test.py to do better factory reset

Fixes project-chip#21736

Testing done:
- Manually tested the fix of TC-RR-1.1 on ESP32 and Linux
- CI tests of the above is automated
  • Loading branch information
tcarmelveilleux committed Aug 31, 2022
1 parent 3be848d commit 3929384
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 5 deletions.
7 changes: 3 additions & 4 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -282,10 +282,9 @@ jobs:
timeout-minutes: 40
run: |
scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"'
# the below tests are broken, enable them when they are fixed. Suspect hermetic issues for now
# scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"'
# scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"'
# scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py"'
scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"'
scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"'
scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true"'
- name: Uploading core files
uses: actions/upload-artifact@v2
if: ${{ failure() && !env.ACT }}
Expand Down
18 changes: 18 additions & 0 deletions scripts/tests/run_python_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import pathlib
import pty
import queue
import re
import shlex
import signal
import subprocess
Expand Down Expand Up @@ -76,10 +77,27 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st
@click.option("--script-gdb", is_flag=True, help='Run script through gdb')
def main(app: str, factoryreset: bool, app_args: str, script: str, script_args: str, script_gdb: bool):
if factoryreset:
# Remove native app config
retcode = subprocess.call("rm -rf /tmp/chip* /tmp/repl*", shell=True)
if retcode != 0:
raise Exception("Failed to remove /tmp/chip* for factory reset.")

# Remove native app KVS if that was used
kvs_match = re.match(r"--KVS (?P<kvs_path>[^ ]+)", app_args)
if kvs_match:
kvs_path_to_remove = kvs_match.group("kvs_path")
retcode = subprocess.call("rm -f %s" % kvs_path_to_remove, shell=True)
if retcode != 0:
raise Exception("Failed to remove %s for factory reset." % kvs_path_to_remove)

# Remove Python test admin storage if provided
storage_match = re.match(r"--storage-path (?P<storage_path>[^ ]+)", app_args)
if storage_match:
storage_path_to_remove = storage_match.group("storage_path")
retcode = subprocess.call("rm -f %s" % storage_path_to_remove, shell=True)
if retcode != 0:
raise Exception("Failed to remove %s for factory reset." % storage_path_to_remove)

coloredlogs.install(level='INFO')

log_queue = queue.Queue()
Expand Down
2 changes: 1 addition & 1 deletion src/python_testing/matter_testing_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -686,7 +686,7 @@ def default_matter_test_main(argv=None, **kwargs):
matter_test_config = parse_matter_test_args(argv)

# Allow override of command line from optional arguments
if matter_test_config.controller_cat_tags is None and "controller_cat_tags" in kwargs:
if not matter_test_config.controller_cat_tags and "controller_cat_tags" in kwargs:
matter_test_config.controller_cat_tags = kwargs["controller_cat_tags"]

# Find the test class in the test script.
Expand Down

0 comments on commit 3929384

Please sign in to comment.