Newly semi/automated tests (#30090)

TC-REFALM-2.1
TC-TCCM-3.2
TC-TCCM-3.3
Modified tests:
TC-MOD-3.1
TC-MOD-3.2
TC-MOD-3.3
TC-RVCRUNM-3.1

examples/darwin-framework-tool/templates/tests/ciTests.json

@@ -54,6 +54,9 @@
         "Disabled because darwin-framework-tool does not support EqualityCommands pseudo-cluster",
         "Test_TC_S_2_2",
         "Test_TC_TCCM_3_1",
+        "Test_TC_TCCM_3_2",
+        "Test_TC_TCCM_3_3",
+        "Test_TC_RVCRUNM_3_1",
         "Test_TC_TCTL_2_1",
         "Disabled because darwin-framework-tool does not support constraints arithmetic operations",
         "Test_TC_FLW_2_1",

scripts/tests/chiptest/__init__.py

@@ -188,6 +188,8 @@ def _GetChipReplUnsupportedTests() -> Set[str]:
         "Test_TC_RVCRUNM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
         "Test_TC_RVCCLEANM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
         "Test_TC_TCCM_3_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
+        "Test_TC_TCCM_3_2.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
+        "Test_TC_TCCM_3_3.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
         "Test_TC_TCTL_2_1.yaml",            # chip-repl does not support EqualityCommands pseudo-cluster
         # chip-repl and chip-tool disagree on what the YAML here should look like: https://github.com/project-chip/connectedhomeip/issues/29110
         "TestClusterMultiFabric.yaml",

src/app/tests/suites/certification/Test_TC_MOD_3_1.yaml

@@ -31,31 +31,31 @@ config:
         defaultValue: 5000
 
 tests:
-    - label: "Wait for the commissioned device to be retrieved"
+    - label: "Step 0:Wait for the commissioned device to be retrieved"
       cluster: "DelayCommands"
       command: "WaitForCommissionee"
       arguments:
           values:
               - name: "nodeId"
                 value: nodeId
 
-    - label: "TH reads the OnMode attribute from the DUT"
+    - label: "Step 1: TH reads the OnMode attribute from the DUT"
       PICS: MOD.S.A0005 && MOD.S.F00
       command: "readAttribute"
       attribute: "OnMode"
       response:
           saveAs: on_mode_dut
 
-    - label: "TH reads the CurrentMode attribute from the DUT"
+    - label: "Step 2a: TH reads the CurrentMode attribute from the DUT"
       PICS: MOD.S.A0003 && MOD.S.F00
       command: "readAttribute"
       attribute: "CurrentMode"
       response:
           saveAs: old_current_mode_dut
 
     - label:
-          "If the OnMode and CurrentMode attributes have the same value, proceed
-          to step 2c, Otherwise proceed to step 3a"
+          "Step 2b: If the OnMode and CurrentMode attributes have the same
+          value, proceed to step 2c, Otherwise proceed to step 3a"
       PICS: MOD.S.A0003 && MOD.S.A0005 && MOD.S.F00
       cluster: "EqualityCommands"
       command: "UnsignedNumberEquals"
@@ -70,7 +70,7 @@ tests:
                 - name: "Equals"
                   saveAs: IsExpectedValue
 
-    - label: "TH reads the SupportedModes attribute from the DUT"
+    - label: "Step 2c: TH reads the SupportedModes attribute from the DUT"
       runIf: IsExpectedValue
       PICS: MOD.S.A0005 && MOD.S.F00
       command: "readAttribute"
@@ -80,9 +80,9 @@ tests:
               type: list
 
     - label:
-          "Select a value from the list in step 2c with a different value than
-          the onMode value read in step 1. TH sends a ChangeToMode command to
-          the TH with the selected value."
+          "Step 2d: Select a value from the list in step 2c with a different
+          value than the onMode value read in step 1. TH sends a ChangeToMode
+          command to the TH with the selected value."
       runIf: IsExpectedValue
       PICS: MOD.S.A0002 && MOD.S.C00.Rsp && MOD.S.F00
       command: "ChangeToMode"
@@ -91,12 +91,12 @@ tests:
               - name: "NewMode"
                 value: NewMode
 
-    - label: "TH sends an Off command to the DUT"
+    - label: "Step 3a: TH sends an Off command to the DUT"
       PICS: OO.S.C00.Rsp && MOD.S.F00
       cluster: "On/Off"
       command: "Off"
 
-    - label: "TH sends an On command to the DUT"
+    - label: "Step 3b: TH sends an On command to the DUT"
       PICS: OO.S.C01.Rsp && MOD.S.F00
       cluster: "On/Off"
       command: "On"
@@ -109,7 +109,7 @@ tests:
               - name: "ms"
                 value: WaitTime
 
-    - label: "TH reads the CurrentMode attribute from the DUT"
+    - label: "Step 3c: TH reads the CurrentMode attribute from the DUT"
       PICS: MOD.S.A0003 && MOD.S.F00
       command: "readAttribute"
       attribute: "CurrentMode"

src/app/tests/suites/certification/Test_TC_MOD_3_2.yaml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# Auto-generated scripts for harness use only, please review before automation. The endpoints and cluster names are currently set to default
 
 name: 80.5.2. [TC-MOD-3.2] StartUpMode verification with DUT as Server
 
@@ -28,31 +27,31 @@ config:
         defaultValue: 7
 
 tests:
-    - label: "Wait for the commissioned device to be retrieved"
+    - label: "Step 0: Wait for the commissioned device to be retrieved"
       cluster: "DelayCommands"
       command: "WaitForCommissionee"
       arguments:
           values:
               - name: "nodeId"
                 value: nodeId
 
-    - label: "TH reads the StartUpMode attribute from the DUT"
+    - label: "Step 1: TH reads the StartUpMode attribute from the DUT"
       PICS: MOD.S.A0004
       command: "readAttribute"
       attribute: "StartUpMode"
       response:
           saveAs: startup_mode_dut
 
-    - label: "TH reads the CurrentMode attribute from the DUT"
+    - label: "Step 2a: TH reads the CurrentMode attribute from the DUT"
       PICS: MOD.S.A0003
       command: "readAttribute"
       attribute: "CurrentMode"
       response:
           saveAs: old_current_mode_dut
 
     - label:
-          "If startup_mode_dut is equal to old_current_mode_dut proceed to step
-          4. Else proceed to step 6"
+          "Step 2b: If the StartUpMode and CurrentMode attributes have the same
+          value, proceed to step 2c, Otherwise proceed to step 3a."
       PICS: MOD.S.A0003 && MOD.S.A0004
       cluster: "EqualityCommands"
       command: "UnsignedNumberEquals"
@@ -67,7 +66,7 @@ tests:
                 - name: "Equals"
                   saveAs: IsExpectedValue
 
-    - label: "TH reads the SupportedModes attribute from the DUT"
+    - label: "Step 2c: TH reads the SupportedModes attribute from the DUT"
       runIf: IsExpectedValue
       PICS: MOD.S.A0002
       command: "readAttribute"
@@ -77,9 +76,9 @@ tests:
               type: list
 
     - label:
-          "Select a value from the list in step 2c with a different value than
-          the StartUpMode value read in step 1. TH sends a ChangeToMode command
-          to the DUT with the selected value."
+          "Step 2d: Select a value from the list in step 2c with a different
+          value than the StartUpMode value read in step 1. TH sends a
+          ChangeToMode command to the DUT with the selected value."
       runIf: IsExpectedValue
       PICS: MOD.S.A0002 && MOD.S.C00.Rsp
       command: "ChangeToMode"
@@ -88,7 +87,7 @@ tests:
               - name: "NewMode"
                 value: NewMode
 
-    - label: "Physically power cycle the device."
+    - label: "Step 3a: Physically power cycle the device."
       verification: |
           Physically power cycle the device.
       cluster: "LogCommands"
@@ -101,15 +100,15 @@ tests:
               - name: "expectedValue"
                 value: "y"
 
-    - label: "Commission device after reboot"
+    - label: "Wait for the commissioned device to be retrieved"
       cluster: "DelayCommands"
       command: "WaitForCommissionee"
       arguments:
           values:
               - name: "nodeId"
                 value: nodeId
 
-    - label: "TH reads the CurrentMode attribute from the DUT"
+    - label: "Step 3b: TH reads the CurrentMode attribute from the DUT"
       PICS: MOD.S.A0003 && MOD.S.F00 && PICS_USER_PROMPT
       command: "readAttribute"
       attribute: "CurrentMode"

src/app/tests/suites/certification/Test_TC_MOD_3_3.yaml

@@ -127,7 +127,7 @@ tests:
               - name: "expectedValue"
                 value: "y"
 
-    - label: "Commission device after reboot"
+    - label: "Wait for the commissioned device to be retrieved"
       cluster: "DelayCommands"
       command: "WaitForCommissionee"
       arguments:

src/app/tests/suites/certification/Test_TC_REFALM_2_1.yaml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# Auto-generated scripts for harness use only, please review before automation. The endpoints and cluster names are currently set to default
 
 name: 222.2.1. [TC-REFALM-2.1] Attributes with DUT as Server
 
@@ -20,62 +19,58 @@ PICS:
 
 config:
     nodeId: 0x12344321
-    cluster: "Basic Information"
-    endpoint: 0
+    cluster: "Refrigerator Alarm"
+    endpoint: 1
 
 tests:
-    - label: "Note"
-      verification: |
-          This is a simulated example log for instructional purposes only. In real scenarios, the actual log may vary depending on the feature implementation in Reference App.
-      disabled: true
-
     - label:
           "Step 1: Commission DUT to TH (can be skipped if done in a preceding
           test)"
-      verification: |
-
-      disabled: true
+      cluster: "DelayCommands"
+      command: "WaitForCommissionee"
+      arguments:
+          values:
+              - name: "nodeId"
+                value: nodeId
 
     - label:
           "Step 2: Ensure that the door alarm is not locally suppressed and the
           door is closed"
       verification: |
           Ensure that the door alarm is not locally suppressed and the door is closed
-      disabled: true
+      cluster: "LogCommands"
+      command: "UserPrompt"
+      PICS: PICS_USER_PROMPT
+      arguments:
+          values:
+              - name: "message"
+                value: "Please enter 'y' for success"
+              - name: "expectedValue"
+                value: "y"
 
-    - label: "Step 3: TH reads from the DUT the Mask attribute"
+    - label: "Step 3:TH reads from the DUT the Mask attribute"
       PICS: REFALM.S.A0000
-      verification: |
-          ./chip-tool refrigeratoralarm read mask 1 1
-          On TH(chip-tool), Verify that the DUT response contains a 32-bit value with bit 0 set to 1.
-          [1688447677.832882][4229:4231] CHIP:TOO: Endpoint: 1 Cluster: 0x0000_0057 Attribute 0x0000_0000 DataVersion: 1795162772
-          [1688447677.833165][4229:4231] CHIP:TOO:   Mask: 1
-          [1688447677.833494][4229:4231] CHIP:EM: <<< [E:27636i S:5449 M:199797248 (Ack:151091416)] (S) Msg TX to 1:0000000000000001 [10DB] --- Type 0000:10 (SecureChannel:StandaloneAck)
-      disabled: true
+      command: "readAttribute"
+      attribute: "Mask"
+      response:
+          constraints:
+              type: bitmap32
+              hasMasksSet: [0x1]
 
-    - label: "Step 4: TH reads from the DUT the State attribute"
+    - label: "Step 4:TH reads from the DUT the State attribute"
       PICS: REFALM.S.A0002
-      verification: |
-          ./chip-tool refrigeratoralarm read state 1 1
-          On TH(chip-tool), Verify that the DUT response contains a 32-bit value with bit 0 set to 0.
-
-          [1688447820.603249][4247:4249] CHIP:TOO: Endpoint: 1 Cluster: 0x0000_0057 Attribute 0x0000_0002 DataVersion: 1795162772
-          [1688447820.603415][4247:4249] CHIP:TOO:   State: 0
-          [1688447820.603708][4247:4249] CHIP:EM: <<< [E:2198i S:4260 M:156565261 (Ack:46517349)] (S) Msg TX to 1:0000000000000001 [10DB] --- Type 0000:10 (SecureChannel:StandaloneAck)
-      disabled: true
+      command: "readAttribute"
+      attribute: "State"
+      response:
+          constraints:
+              type: bitmap32
+              hasMasksClear: [0x1]
 
-    - label: "Step 5: TH reads from the DUT the Supported attribute"
+    - label: "Step 5:TH reads from the DUT the Supportedask attribute"
       PICS: REFALM.S.A0003
-      verification: |
-          ./chip-tool refrigeratoralarm read supported 1 1
-          On TH(chip-tool), Verify that the DUT response contains a 32-bit value with bit 0 set to 1.
-
-          [1689677642.708638][18413:18415] CHIP:DMG:         SuppressResponse = true,
-          [1689677642.708645][18413:18415] CHIP:DMG:         InteractionModelRevision = 1
-          [1689677642.708651][18413:18415] CHIP:DMG: }
-          [1689677642.708726][18413:18415] CHIP:TOO: Endpoint: 1 Cluster: 0x0000_0057 Attribute 0x0000_0003 DataVersion: 1517282962
-          [1689677642.708773][18413:18415] CHIP:TOO:   Supported: 1
-          [1689677642.708845][18413:18415] CHIP:EM: <<< [E:32432i S:53289 M:29829407 (Ack:142651059)] (S) Msg TX to 1:0000000000000001 [BFDE] --- Type 0000:10 (SecureChannel:StandaloneAck)
-          [1689677642.708859][18413:18415] CHIP:IN: (S) Sending msg 29829407 on secure session with LSID: 53289
-          [1689677642.708900][18413:18415] CHIP:EM: Flushed pending ack for MessageCounter:142651059 on exchange 32432i
-      disabled: true
+      command: "readAttribute"
+      attribute: "Supported"
+      response:
+          constraints:
+              type: bitmap32
+              hasMasksSet: [0x1]

src/app/tests/suites/certification/Test_TC_RVCRUNM_3_1.yaml

@@ -17,6 +17,8 @@ name: 87.4.1. [TC-RVCRUNM-3.1] On Mode functionality with DUT as Server
 PICS:
     - RVCRUNM.S.A0003
     - RVCRUNM.S.F00
+    - OO.S.C00.Rsp
+    - OO.S.C01.Rsp
 
 config:
     nodeId: 0x12344321