Re-enable TC-RR-1.1 in CI (project-chip#22292)

* Re-enable TC-RR-1.1 in CI

- A regression had occured on RR-1.1 in master before CI was attempted
  in project-chip#22143 which prevented the CI test to be enabled

This PR:

- Fixes TC-RR-1.1
- Enables TC-RR-1.1, TC-SC-3.6 and TC-DA-1.7 to keep them fresh and
  improve integration test coverage (TC-RR-1.1 tests a lot)
- Improve run_python_test.py to do better factory reset

Fixes project-chip#21736

Testing done:
- Manually tested the fix of TC-RR-1.1 on ESP32 and Linux
- CI tests of the above is automated

* Attempt to pass CI

* Debug CI

* More debug of CI

* More CI fixes

* Fix more CI

.github/workflows/tests.yaml

@@ -243,7 +243,7 @@ jobs:
         container:
             image: connectedhomeip/chip-build:0.5.91
             options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0
-                net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1"
+                net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0"
 
         steps:
             - name: Checkout
@@ -273,19 +273,18 @@ jobs:
                   scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env'
                   ./scripts/run_in_build_env.sh \
                    "./scripts/build/build_examples.py \
-                      --target linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test \
+                      --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test \
                       --target linux-x64-python-bindings \
                       build \
                       --copy-artifacts-to objdir-clone \
                    "
             - name: Run Tests
-              timeout-minutes: 40
+              timeout-minutes: 10
               run: |
-                  scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"'
-              # the below tests are broken, enable them when they are fixed. Suspect hermetic issues for now
-              # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"'
-              # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"'
-              # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app  --factoryreset --app-args "--KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py"'
+                  scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"'
+                  scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"'
+                  scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"'
+                  scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app  --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true"'
             - name: Uploading core files
               uses: actions/upload-artifact@v2
               if: ${{ failure() && !env.ACT }}

scripts/tests/run_python_test.py

@@ -22,6 +22,7 @@
 import pathlib
 import pty
 import queue
+import re
 import shlex
 import signal
 import subprocess
@@ -76,10 +77,32 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st
 @click.option("--script-gdb", is_flag=True, help='Run script through gdb')
 def main(app: str, factoryreset: bool, app_args: str, script: str, script_args: str, script_gdb: bool):
     if factoryreset:
+        # Remove native app config
         retcode = subprocess.call("rm -rf /tmp/chip* /tmp/repl*", shell=True)
         if retcode != 0:
             raise Exception("Failed to remove /tmp/chip* for factory reset.")
 
+        print("Contents of test directory: %s" % os.getcwd())
+        print(subprocess.check_output(["ls -l"], shell=True).decode('us-ascii'))
+
+        # Remove native app KVS if that was used
+        kvs_match = re.search(r"--KVS (?P<kvs_path>[^ ]+)", app_args)
+        if kvs_match:
+            kvs_path_to_remove = kvs_match.group("kvs_path")
+            retcode = subprocess.call("rm -f %s" % kvs_path_to_remove, shell=True)
+            print("Trying to remove KVS path %s" % kvs_path_to_remove)
+            if retcode != 0:
+                raise Exception("Failed to remove %s for factory reset." % kvs_path_to_remove)
+
+        # Remove Python test admin storage if provided
+        storage_match = re.search(r"--storage-path (?P<storage_path>[^ ]+)", script_args)
+        if storage_match:
+            storage_path_to_remove = storage_match.group("storage_path")
+            retcode = subprocess.call("rm -f %s" % storage_path_to_remove, shell=True)
+            print("Trying to remove storage path %s" % storage_path_to_remove)
+            if retcode != 0:
+                raise Exception("Failed to remove %s for factory reset." % storage_path_to_remove)
+
     coloredlogs.install(level='INFO')
 
     log_queue = queue.Queue()

src/python_testing/matter_testing_support.py

@@ -686,7 +686,7 @@ def default_matter_test_main(argv=None, **kwargs):
     matter_test_config = parse_matter_test_args(argv)
 
     # Allow override of command line from optional arguments
-    if matter_test_config.controller_cat_tags is None and "controller_cat_tags" in kwargs:
+    if not matter_test_config.controller_cat_tags and "controller_cat_tags" in kwargs:
         matter_test_config.controller_cat_tags = kwargs["controller_cat_tags"]
 
     # Find the test class in the test script.