From 789442898b80b7f937a7e95e357b1003edbcfb5d Mon Sep 17 00:00:00 2001 From: Tennessee Carmel-Veilleux Date: Wed, 31 Aug 2022 21:54:50 -0400 Subject: [PATCH] Re-enable TC-RR-1.1 in CI (#22292) * Re-enable TC-RR-1.1 in CI - A regression had occured on RR-1.1 in master before CI was attempted in #22143 which prevented the CI test to be enabled This PR: - Fixes TC-RR-1.1 - Enables TC-RR-1.1, TC-SC-3.6 and TC-DA-1.7 to keep them fresh and improve integration test coverage (TC-RR-1.1 tests a lot) - Improve run_python_test.py to do better factory reset Fixes #21736 Testing done: - Manually tested the fix of TC-RR-1.1 on ESP32 and Linux - CI tests of the above is automated * Attempt to pass CI * Debug CI * More debug of CI * More CI fixes * Fix more CI --- .github/workflows/tests.yaml | 15 ++++++------- scripts/tests/run_python_test.py | 23 ++++++++++++++++++++ src/python_testing/matter_testing_support.py | 2 +- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ffac7539ec7936..7e45a49844c031 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -243,7 +243,7 @@ jobs: container: image: connectedhomeip/chip-build:0.5.91 options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 - net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1" + net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0" steps: - name: Checkout @@ -273,19 +273,18 @@ jobs: scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env' ./scripts/run_in_build_env.sh \ "./scripts/build/build_examples.py \ - --target linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test \ + --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test \ --target linux-x64-python-bindings \ build \ --copy-artifacts-to objdir-clone \ " - name: Run Tests - timeout-minutes: 40 + timeout-minutes: 10 run: | - scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"' - # the below tests are broken, enable them when they are fixed. Suspect hermetic issues for now - # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"' - # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"' - # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py"' + scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"' + scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' + scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' + scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true"' - name: Uploading core files uses: actions/upload-artifact@v2 if: ${{ failure() && !env.ACT }} diff --git a/scripts/tests/run_python_test.py b/scripts/tests/run_python_test.py index 421e5f46683eb1..e13d9b5edcd905 100755 --- a/scripts/tests/run_python_test.py +++ b/scripts/tests/run_python_test.py @@ -22,6 +22,7 @@ import pathlib import pty import queue +import re import shlex import signal import subprocess @@ -76,10 +77,32 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st @click.option("--script-gdb", is_flag=True, help='Run script through gdb') def main(app: str, factoryreset: bool, app_args: str, script: str, script_args: str, script_gdb: bool): if factoryreset: + # Remove native app config retcode = subprocess.call("rm -rf /tmp/chip* /tmp/repl*", shell=True) if retcode != 0: raise Exception("Failed to remove /tmp/chip* for factory reset.") + print("Contents of test directory: %s" % os.getcwd()) + print(subprocess.check_output(["ls -l"], shell=True).decode('us-ascii')) + + # Remove native app KVS if that was used + kvs_match = re.search(r"--KVS (?P[^ ]+)", app_args) + if kvs_match: + kvs_path_to_remove = kvs_match.group("kvs_path") + retcode = subprocess.call("rm -f %s" % kvs_path_to_remove, shell=True) + print("Trying to remove KVS path %s" % kvs_path_to_remove) + if retcode != 0: + raise Exception("Failed to remove %s for factory reset." % kvs_path_to_remove) + + # Remove Python test admin storage if provided + storage_match = re.search(r"--storage-path (?P[^ ]+)", script_args) + if storage_match: + storage_path_to_remove = storage_match.group("storage_path") + retcode = subprocess.call("rm -f %s" % storage_path_to_remove, shell=True) + print("Trying to remove storage path %s" % storage_path_to_remove) + if retcode != 0: + raise Exception("Failed to remove %s for factory reset." % storage_path_to_remove) + coloredlogs.install(level='INFO') log_queue = queue.Queue() diff --git a/src/python_testing/matter_testing_support.py b/src/python_testing/matter_testing_support.py index c50270e8aa2c2f..98ffa4bf064e0e 100644 --- a/src/python_testing/matter_testing_support.py +++ b/src/python_testing/matter_testing_support.py @@ -686,7 +686,7 @@ def default_matter_test_main(argv=None, **kwargs): matter_test_config = parse_matter_test_args(argv) # Allow override of command line from optional arguments - if matter_test_config.controller_cat_tags is None and "controller_cat_tags" in kwargs: + if not matter_test_config.controller_cat_tags and "controller_cat_tags" in kwargs: matter_test_config.controller_cat_tags = kwargs["controller_cat_tags"] # Find the test class in the test script.