Add a way to have tests run only in the python-chip-tool driver.

Switches TestGroupKeyManagementCluster to the new setup and enables the disabled parts of it. Fixes #28396
project-chip · Aug 4, 2023 · 240aaf5 · 240aaf5
1 parent 1465733
commit 240aaf5
Show file tree

Hide file tree

Showing 8 changed files with 73 additions and 2,061 deletions.
diff --git a/examples/chip-tool/templates/tests/commands.zapt b/examples/chip-tool/templates/tests/commands.zapt
@@ -36,6 +36,18 @@ public:
     }
 };
 
+class PythonRunnerOnlyTestList : public Command
+{
+public:
+    PythonRunnerOnlyTestList() : Command("list-python-runner-only") {};
+    CHIP_ERROR Run() override
+    {
+        {{#chip_tests "../../../../src/app/tests/suites/pythonRunnerOnlyTests.json" includeAllClusters=true}}printf("{{filename}}\n");{{/chip_tests}}
+
+        return CHIP_NO_ERROR;
+    }
+};
+
 {{>test_cluster tests="../../../../src/app/tests/suites/ciTests.json" credsIssuerConfigArg=true needsWaitDuration=true}}
 {{>test_cluster tests="../../../../src/app/tests/suites/manualTests.json" credsIssuerConfigArg=true needsWaitDuration=true}}
 
@@ -49,6 +61,7 @@ void registerCommandsTests(Commands & commands, CredentialIssuerCommands * creds
 #if CONFIG_ENABLE_YAML_TESTS
         make_unique<TestList>(),
         make_unique<ManualTestList>(),
+        make_unique<PythonRunnerOnlyTestList>(),
       {{#chip_tests "../../../../src/app/tests/suites/ciTests.json" includeAllClusters=true}}
         make_unique<{{filename}}Suite>(credsIssuerConfig),
       {{/chip_tests}}

diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py
@@ -185,19 +185,23 @@ def target_for_name(name: str):
     return TestTarget.ALL_CLUSTERS
 
 
-def tests_with_command(chip_tool: str, is_manual: bool):
+def tests_with_command(chip_tool: str, is_manual: bool, is_chip_tool_python_only: bool = False):
     """Executes `chip_tool` binary to see what tests are available, using cmd
     to get the list.
     """
     cmd = "list"
     if is_manual:
         cmd += "-manual"
+    elif is_chip_tool_python_only:
+        cmd += "-python-runner-only"
 
     result = subprocess.run([chip_tool, "tests", cmd], capture_output=True)
 
     test_tags = set()
     if is_manual:
         test_tags.add(TestTag.MANUAL)
+    if is_chip_tool_python_only:
+        test_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)
 
     in_development_tests = [s.replace(".yaml", "") for s in _GetInDevelopmentTests()]
 
@@ -260,6 +264,9 @@ def AllChipToolTests(chip_tool: str):
     for test in tests_with_command(chip_tool, is_manual=True):
         yield test
 
+    for test in tests_with_command(chip_tool, is_manual=False, is_chip_tool_python_only=True):
+        yield test
+
 
 __all__ = [
     "TestTarget",

diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py
@@ -217,6 +217,7 @@ class TestTag(Enum):
     SLOW = auto()            # test uses Sleep and is generally slow (>=10s is a typical threshold)
     FLAKY = auto()           # test is considered flaky (usually a bug/time dependent issue)
     IN_DEVELOPMENT = auto()  # test may not pass or undergoes changes
+    CHIP_TOOL_PYTHON_ONLY = auto()  # test uses YAML features only supported by the CHIP_TOOL_PYTHON runner.
 
     def to_s(self):
         for (k, v) in TestTag.__members__.items():

diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py
@@ -174,6 +174,9 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
             TestTag.FLAKY,
         }
 
+        if runtime != TestRunTime.CHIP_TOOL_PYTHON:
+            exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)
+
     if 'all' not in target:
         tests = []
         for name in target:

diff --git a/src/app/tests/suites/TestGroupKeyManagementCluster.yaml b/src/app/tests/suites/TestGroupKeyManagementCluster.yaml
@@ -74,30 +74,29 @@ tests:
           constraints:
               minValue: 3
 
-    # TODO(#28396): Re-enabled when CI wouldn't break on this syntax
-    #
-    #   command: "readAttribute"
-    #   attribute: "ClusterRevision"
-    #   response:
-    #       saveAs: ClusterRevisionValue
-    #       constraints:
-    #           minValue: 1
-    #           maxValue: 2
-    #           type: int16u
-
-    # - label: "Creates a new variable to hold if the cluster revision is == 2"
-    #   cluster: EqualityCommands
-    #   command: UnsignedNumberEquals
-    #   arguments:
-    #       values:
-    #           - name: Value1
-    #             value: ClusterRevisionValue
-    #           - name: Value2
-    #             value: 2
-    #   response:
-    #       - values:
-    #             - name: Equals
-    #               saveAs: ClusterIsRevision2
+    - label: "Read the ClusterRevision attribute"
+      command: "readAttribute"
+      attribute: "ClusterRevision"
+      response:
+          saveAs: ClusterRevisionValue
+          constraints:
+              minValue: 1
+              maxValue: 2
+              type: int16u
+
+    - label: "Creates a new variable to hold if the cluster revision is == 2"
+      cluster: EqualityCommands
+      command: UnsignedNumberEquals
+      arguments:
+          values:
+              - name: Value1
+                value: ClusterRevisionValue
+              - name: Value2
+                value: 2
+      response:
+          - values:
+                - name: Equals
+                  saveAs: ClusterIsRevision2
 
     ########### KeySetWrite Epoch0 field validations
     - label: "KeySetWrite with EpochKey0 null fails INVALID_COMMAND"
@@ -161,7 +160,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey0 with length 1 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -183,7 +182,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey0 with length 0 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -204,7 +203,7 @@ tests:
 
     ########### KeySetWrite Epoch1 field validations
     - label: "KeySetWrite with EpochStartTime1 null fails INVALID_COMMAND"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -245,7 +244,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey1 with length 1 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -267,7 +266,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey1 with length 0 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -310,7 +309,7 @@ tests:
     ########### KeySetWrite Epoch2 field validations
     - label: "KeySetWrite with EpochStartTime2 null fails INVALID_COMMAND"
       command: "KeySetWrite"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       arguments:
           values:
               - name: "GroupKeySet"
@@ -350,7 +349,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey2 with length 1 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:
@@ -372,7 +371,7 @@ tests:
     - label:
           "KeySetWrite with EpochKey2 with length 0 != 16 fails with
           CONSTRAINT_ERROR"
-      # runIf: ClusterIsRevision2
+      runIf: ClusterIsRevision2
       command: "KeySetWrite"
       arguments:
           values:

diff --git a/src/app/tests/suites/ciTests.json b/src/app/tests/suites/ciTests.json
@@ -297,7 +297,6 @@
     "Groups": [
         "TestGroupMessaging",
         "TestGroupsCluster",
-        "TestGroupKeyManagementCluster",
         "Test_TC_G_1_1",
         "Test_TC_G_2_1"
     ],

diff --git a/src/app/tests/suites/pythonRunnerOnlyTests.json b/src/app/tests/suites/pythonRunnerOnlyTests.json
@@ -0,0 +1,4 @@
+{
+    "Groups": ["TestGroupKeyManagementCluster"],
+    "collection": ["Groups"]
+}