Add checking for correctness of error codes to YAML tests. (#12306)

For now, uses errorWrongValue in tests where the wrong error code is returned by the SDK.
project-chip · Jun 9, 2022 · f356dc3 · f356dc3
1 parent 15f39e5
commit f356dc3
Show file tree

Hide file tree

Showing 41 changed files with 2,176 additions and 1,565 deletions.
diff --git a/examples/chip-tool/templates/partials/test_cluster.zapt b/examples/chip-tool/templates/partials/test_cluster.zapt
@@ -226,17 +226,23 @@ class {{filename}}: public TestCommand
 
     void {{>failureResponse}}({{>failureArguments}})
     {
-        {{~#unless response.error}}
-        {{#if optional}}(status == EMBER_ZCL_STATUS_UNSUPPORTED_ATTRIBUTE) ? NextTest() : {{/if}}ThrowFailureResponse();
+        {{#if response.error}}
+          VerifyOrReturn(CheckValue("status", status, {{response.error}}));
+          {{#unless async}}NextTest();{{/unless}}
+        {{else if response.errorWrongValue}}
+          VerifyOrReturn(CheckConstraintNotValue("status", status, 0));
+          {{#unless async}}NextTest();{{/unless}}
         {{else}}
-        {{#unless async}}NextTest();{{/unless}}
-        {{/unless}}
+          {{#if optional}}(status == EMBER_ZCL_STATUS_UNSUPPORTED_ATTRIBUTE) ? NextTest() : {{/if}}ThrowFailureResponse();
+        {{/if}}
     }
 
     void {{>successResponse}}({{>successArguments}})
     {
         {{~#if response.error}}
-        ThrowSuccessResponse();
+          ThrowSuccessResponse();
+        {{else if response.errorWrongValue}}
+          ThrowSuccessResponse();
         {{else}}
           {{! This block can be removed if there is a mechanism to remove the report handler}}
           {{#if isWaitForReport}}

diff --git a/src/app/tests/suites/TestCluster.yaml b/src/app/tests/suites/TestCluster.yaml
@@ -25,7 +25,7 @@ tests:
     - label: "Send Test Not Handled Command"
       command: "testNotHandled"
       response:
-          error: 1
+          error: 0x85 # INVALID_COMMAND
 
     - label: "Send Test Specific Command"
       command: "testSpecific"
@@ -56,7 +56,7 @@ tests:
               - name: "arg2"
                 value: 6
       response:
-          error: 1
+          error: 0x85 # INVALID_COMMAND
 
     # Tests for Boolean attribute
 
@@ -624,7 +624,7 @@ tests:
       arguments:
           value: "TestValueLongerThan10"
       response:
-          error: true
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute OCTET_STRING"
       command: "readAttribute"
@@ -690,7 +690,7 @@ tests:
       arguments:
           value: "☉TestValueLongerThan10☉"
       response:
-          error: true
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute CHAR_STRING"
       command: "readAttribute"
@@ -814,8 +814,17 @@ tests:
       command: "test"
       endpoint: 200
       response:
-          # No cluster on that endpoint, so expect an error.
-          error: 1
+          # No such endpoint, so expect an error.
+          # SDK returning wrong error code here so far.
+          errorWrongValue: 0x7F # UNSUPPORTED_ENDPOINT
+
+    - label: "Send Test Command to unsupported cluster"
+      command: "test"
+      endpoint: 0
+      response:
+          # No TestCluster on that, so expect an error.
+          # SDK returning wrong error code here so far.
+          errorWrongValue: 0xC3 # UNSUPPORTED_CLUSTER
 
     # Tests for vendor id
 
@@ -1471,7 +1480,7 @@ tests:
       arguments:
           value: 255
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_BITMAP8 unchanged Value"
       command: "readAttribute"
@@ -1511,7 +1520,7 @@ tests:
       arguments:
           value: 65535
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_BITMAP16 unchanged Value"
       command: "readAttribute"
@@ -1551,7 +1560,7 @@ tests:
       arguments:
           value: 4294967295
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_BITMAP32 unchanged Value"
       command: "readAttribute"
@@ -1591,7 +1600,7 @@ tests:
       arguments:
           value: "18446744073709551615"
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_BITMAP64 unchanged Value"
       command: "readAttribute"
@@ -1631,7 +1640,7 @@ tests:
       arguments:
           value: 255
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT8U unchanged Value"
       command: "readAttribute"
@@ -1671,7 +1680,7 @@ tests:
       arguments:
           value: 65535
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT16U unchanged Value"
       command: "readAttribute"
@@ -1711,7 +1720,7 @@ tests:
       arguments:
           value: 4294967295
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT32U unchanged Value"
       command: "readAttribute"
@@ -1751,7 +1760,7 @@ tests:
       arguments:
           value: "18446744073709551615"
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT64U unchanged Value"
       command: "readAttribute"
@@ -1791,7 +1800,7 @@ tests:
       arguments:
           value: -128
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT8S unchanged Value"
       command: "readAttribute"
@@ -1831,7 +1840,7 @@ tests:
       arguments:
           value: -32768
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT16S unchanged Value"
       command: "readAttribute"
@@ -1871,7 +1880,7 @@ tests:
       arguments:
           value: -2147483648
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT32S unchanged Value"
       command: "readAttribute"
@@ -1914,7 +1923,7 @@ tests:
           # work around that.
           value: "-9223372036854775807LL - 1"
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_INT64S unchanged Value"
       command: "readAttribute"
@@ -1954,7 +1963,7 @@ tests:
       arguments:
           value: 255
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_ENUM8 unchanged Value"
       command: "readAttribute"
@@ -1994,7 +2003,7 @@ tests:
       arguments:
           value: 65535
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
 
     - label: "Read attribute NULLABLE_ENUM16 unchanged Value"
       command: "readAttribute"
@@ -2103,9 +2112,18 @@ tests:
       response:
           value: ""
 
-    - label: "Read nonexistent attribute."
+    - label: "Read attribute from nonexistent endpoint."
       endpoint: 200
       command: "readAttribute"
       attribute: "list_int8u"
       response:
-          error: 0x86 # UNSUPPORTED_ATTRIBUTE
+          # SDK returning wrong error code here so far.
+          errorWrongValue: 0x7F # UNSUPPORTED_ENDPOINT
+
+    - label: "Read attribute from nonexistent cluster."
+      endpoint: 0
+      command: "readAttribute"
+      attribute: "list_int8u"
+      response:
+          # SDK returning wrong error code here so far.
+          errorWrongValue: 0xC3 # UNSUPPORTED_CLUSTER
diff --git a/src/app/tests/suites/TestModeSelectCluster.yaml b/src/app/tests/suites/TestModeSelectCluster.yaml
@@ -80,4 +80,4 @@ tests:
               - name: "NewMode"
                 value: 2
       response:
-          error: 1
+          error: 0x87 # CONSTRAINT_ERROR
diff --git a/src/app/tests/suites/certification/Test_TC_BI_1_1.yaml b/src/app/tests/suites/certification/Test_TC_BI_1_1.yaml
@@ -33,7 +33,7 @@ tests:
       arguments:
           value: 1
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "reads back global attribute: ClusterRevision"
       command: "readAttribute"

diff --git a/src/app/tests/suites/certification/Test_TC_BI_2_1.yaml b/src/app/tests/suites/certification/Test_TC_BI_2_1.yaml
@@ -90,7 +90,7 @@ tests:
       arguments:
           value: 0
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "Reads back the mandatory non-global attribute: StatusFlags"
       command: "readAttribute"
@@ -214,7 +214,7 @@ tests:
       arguments:
           value: 0
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "Reads back the optional non-global attribute: Polarity"
       disabled: true
@@ -273,7 +273,7 @@ tests:
       arguments:
           value: 0
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "Reads back the optional non-global attribute: ApplicationType"
       disabled: true

diff --git a/src/app/tests/suites/certification/Test_TC_BOOL_1_1.yaml b/src/app/tests/suites/certification/Test_TC_BOOL_1_1.yaml
@@ -33,7 +33,7 @@ tests:
       arguments:
           value: 1
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "reads back global attribute: ClusterRevision"
       command: "readAttribute"

diff --git a/src/app/tests/suites/certification/Test_TC_BOOL_2_1.yaml b/src/app/tests/suites/certification/Test_TC_BOOL_2_1.yaml
@@ -39,7 +39,7 @@ tests:
       arguments:
           value: 1
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "Reads back the mandatory non-global attribute: StateValue"
       command: "readAttribute"

diff --git a/src/app/tests/suites/certification/Test_TC_CC_1_1.yaml b/src/app/tests/suites/certification/Test_TC_CC_1_1.yaml
@@ -34,7 +34,7 @@ tests:
       arguments:
           value: 4
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "reads back global attribute: ClusterRevision"
       disabled: true
@@ -57,7 +57,7 @@ tests:
       arguments:
           value: 0
       response:
-          error: 1
+          error: 0x88 # UNSUPPORTED_WRITE
 
     - label: "reads back optional global attribute: FeatureMap"
       disabled: true