From db09a6405a10438ee5a67c2d4d7c1d65c50afcac Mon Sep 17 00:00:00 2001 From: Terence Hampson Date: Tue, 13 Feb 2024 09:33:57 -0500 Subject: [PATCH] Add test steps to IDM_1_4 to validate multiple InvokeResponseMessages (#31979) * Add test steps to IDM_1_4 to validate multiple InvokeResponseMessages * Restyled by autopep8 * Address PR comments and CI issues * rename variable * Fix CI * Fix incorrect merge attempt * Fix CI --------- Co-authored-by: Restyled.io --- .github/workflows/tests.yaml | 2 +- src/python_testing/TC_IDM_1_4.py | 86 +++++++++++++++++++++++++++----- 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index bd9ac282b78a79..af9ee570102fac 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -500,7 +500,7 @@ jobs: scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_FAN_3_5.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-lit-icd-ipv6only-no-ble-no-wifi-tsan-clang-test/lit-icd-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_ICDM_2_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_IDM_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_IDM_1_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json --enable-key 000102030405060708090a0b0c0d0e0f" --script "src/python_testing/TC_IDM_1_4.py" --script-args "--hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:000102030405060708090a0b0c0d0e0f --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_IDM_4_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RVCCLEANM_1_2.py" --script-args "--int-arg PIXIT_ENDPOINT:1 --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' diff --git a/src/python_testing/TC_IDM_1_4.py b/src/python_testing/TC_IDM_1_4.py index 86e49bb3523f42..507b5d74254abf 100644 --- a/src/python_testing/TC_IDM_1_4.py +++ b/src/python_testing/TC_IDM_1_4.py @@ -16,21 +16,16 @@ # import logging -from dataclasses import dataclass import chip.clusters as Clusters -from chip import ChipUtility from chip.exceptions import ChipStackError from chip.interaction_model import InteractionModelError, Status from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main, type_matches from mobly import asserts - -@dataclass -class FakeInvalidBasicInformationCommand(Clusters.BasicInformation.Commands.MfgSpecificPing): - @ChipUtility.classproperty - def must_use_timed_invoke(cls) -> bool: - return False +# If DUT supports `MaxPathsPerInvoke > 1`, additional command line argument +# run with +# --hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY: class TC_IDM_1_4(MatterBaseTest): @@ -44,7 +39,9 @@ def steps_TC_IDM_1_4(self) -> list[TestStep]: TestStep(6, "Verify DUT responds to InvokeRequestMessage containing one valid paths, and one InvokeRequest to unsupported endpoint"), TestStep(7, "Verify DUT responds to InvokeRequestMessage containing two valid paths. One of which requires timed invoke, and TimedRequest in InvokeResponseMessage set to true, but never sending preceding Timed Invoke Action"), TestStep(8, "Verify DUT responds to InvokeRequestMessage containing two valid paths. One of which requires timed invoke, and TimedRequest in InvokeResponseMessage set to true"), - TestStep(9, "Verify DUT capable of responding to request with multiple InvokeResponseMessages")] + TestStep(9, "Verify DUT supports extended Data Model Testing feature in General Diagnostics Cluster"), + TestStep(10, "Verify DUT has TestEventTriggersEnabled attribute set to true in General Diagnostics Cluster"), + TestStep(11, "Verify DUT capable of responding to request with multiple InvokeResponseMessages")] return steps @async_test_body @@ -118,9 +115,14 @@ async def test_TC_IDM_1_4(self): if max_paths_per_invoke == 1: self.skip_all_remaining_steps(3) else: - await self.steps_3_to_9(False) + asserts.assert_true('PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY' in self.matter_test_config.global_test_params, + "PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY must be included on the command line in " + "the --hex-arg flag as PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:, " + "e.g. --hex-arg PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY:000102030405060708090a0b0c0d0e0f") + + await self.remaining_batch_commands_test_steps(False) - async def steps_3_to_9(self, dummy_value): + async def remaining_batch_commands_test_steps(self, dummy_value): dev_ctrl = self.default_controller dut_node_id = self.dut_node_id @@ -247,8 +249,66 @@ async def steps_3_to_9(self, dummy_value): except InteractionModelError: asserts.fail("DUT failed with non-path specific error when path specific error was expected") - # Skipping test until https://github.com/project-chip/connectedhomeip/issues/31434 resolved - self.skip_step(9) + self.step(9) + try: + feature_map = await self.read_single_attribute( + dev_ctrl, + dut_node_id, + endpoint=0, + attribute=Clusters.GeneralDiagnostics.Attributes.FeatureMap + ) + except InteractionModelError: + asserts.fail("DUT failed to respond reading FeatureMap attribute") + has_data_model_test_feature = (feature_map & Clusters.GeneralDiagnostics.Bitmaps.Feature.kDataModelTest) != 0 + asserts.assert_true(has_data_model_test_feature, "DataModelTest Feature is not supported by DUT") + + self.step(10) + try: + test_event_triggers_enabled = await self.read_single_attribute( + dev_ctrl, + dut_node_id, + endpoint=0, + attribute=Clusters.GeneralDiagnostics.Attributes.TestEventTriggersEnabled + ) + except InteractionModelError: + asserts.fail("DUT failed to respond reading TestEventTriggersEnabled attribute") + asserts.assert_true(test_event_triggers_enabled, "Test Event Triggers must be enabled on DUT") + + self.step(11) + enable_key = self.matter_test_config.global_test_params['PIXIT.DGGEN.TEST_EVENT_TRIGGER_KEY'] + endpoint = 0 + command = Clusters.GeneralDiagnostics.Commands.PayloadTestRequest( + enableKey=enable_key, + value=ord('A'), + count=800 + ) + invoke_request_1 = Clusters.Command.InvokeRequestInfo(endpoint, command) + + command = Clusters.OperationalCredentials.Commands.CertificateChainRequest( + Clusters.OperationalCredentials.Enums.CertificateChainTypeEnum.kDACCertificate) + invoke_request_2 = Clusters.Command.InvokeRequestInfo(endpoint, command) + + try: + test_only_result = await dev_ctrl.TestOnlySendBatchCommands(dut_node_id, [invoke_request_1, invoke_request_2]) + except InteractionModelError: + asserts.fail("DUT failed to respond to batch commands, where response is expected to be too large to fit in a single ResponseMessage") + + responses = test_only_result.Responses + # This check is validating the number of InvokeResponses we got + asserts.assert_equal(len(responses), 2, "Unexpected number of InvokeResponses sent back from DUT") + asserts.assert_true(type_matches( + responses[0], Clusters.GeneralDiagnostics.Commands.PayloadTestResponse), "Unexpected return type for first InvokeRequest") + asserts.assert_true(type_matches( + responses[1], Clusters.OperationalCredentials.Commands.CertificateChainResponse), "Unexpected return type for second InvokeRequest") + logging.info("DUT successfully responded to a InvokeRequest action with two valid commands") + + asserts.assert_equal(responses[0].payload, b'A' * 800, "Expect response to match for count == 800") + # If this assert below fails then some assumptions we were relying on are now no longer true. + # This check is validating the number of InvokeResponsesMessages we got. This is different then the earlier + # `len(responses)` check as you can have multiple InvokeResponses in a single message. But this test step + # is explicitly making sure that we recieved multiple ResponseMessages. + asserts.assert_greater_equal(test_only_result.ResponseMessageCount, 2, + "DUT was expected to send multiple InvokeResponseMessages") if __name__ == "__main__":