From 1001996b10caf97268a425ecf673ab7b13bab793 Mon Sep 17 00:00:00 2001 From: C Freeman Date: Tue, 9 Jul 2024 04:57:38 -0400 Subject: [PATCH] IDM-10.4: check that CI PICS is not on for cert (#33817) * IDM-10.1: check that CI PICS is not on for cert Also adding a test of the test to prove that this works. Added attribute tests as well. * Restyled by autopep8 * Restyled by isort * linter * missed one * whoopsie-doodles, changed the wrong one * add to tests.yaml --------- Co-authored-by: Restyled.io --- .github/workflows/tests.yaml | 1 + src/python_testing/TC_pics_checker.py | 19 ++- .../TestMatterTestingSupport.py | 2 +- src/python_testing/matter_testing_support.py | 26 ++-- .../example_pics_xml_basic_info.xml | 2 +- .../test_testing/test_IDM_10_4.py | 126 ++++++++++++++++++ 6 files changed, 158 insertions(+), 18 deletions(-) create mode 100644 src/python_testing/test_testing/test_IDM_10_4.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a8bdfdcdc2c1ec..c77df4c1eed72d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -579,6 +579,7 @@ jobs: scripts/run_in_python_env.sh out/venv './scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py' scripts/run_in_python_env.sh out/venv './src/python_testing/test_testing/test_TC_ICDM_2_1.py' scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/TestIdChecks.py' + scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_IDM_10_4.py' - name: Uploading core files uses: actions/upload-artifact@v4 diff --git a/src/python_testing/TC_pics_checker.py b/src/python_testing/TC_pics_checker.py index 4e503edd101a01..f63628fd4eb426 100644 --- a/src/python_testing/TC_pics_checker.py +++ b/src/python_testing/TC_pics_checker.py @@ -20,7 +20,7 @@ from basic_composition_support import BasicCompositionTests from global_attribute_ids import GlobalAttributeIds from matter_testing_support import (AttributePathLocation, ClusterPathLocation, CommandPathLocation, FeaturePathLocation, - MatterBaseTest, TestStep, async_test_body, default_matter_test_main) + MatterBaseTest, ProblemLocation, TestStep, async_test_body, default_matter_test_main) from mobly import asserts from pics_support import accepted_cmd_pics_str, attribute_pics_str, feature_pics_str, generated_cmd_pics_str from spec_parsing_support import build_xml_clusters @@ -88,11 +88,12 @@ def _add_pics_for_lists(self, cluster_id: int, attribute_id_of_element_list: Glo def steps_TC_IDM_10_4(self): return [TestStep(1, "TH performs a wildcard read of all attributes on the endpoint under test"), - TestStep(2, "For every standard cluster: If the cluster is present on the endpoint, ensure the server-side PICS code for the cluster is present in the PICS file (e.g. OO.S for On/Off cluster).If the cluster is not present on the endpoint, ensure the cluster server PICS code is not present in the PICS file."), - TestStep(3, "For every standard cluster, for every attribute in the cluster:If the cluster is present on the endpoint and the attribute ID is present in the AttributeList global attribute within the cluster, ensure the server-side PICS code for the attribute is present in the PICS file (e.g. OO.S.A000 for On/Off cluster’s OnOff attribute).Otherwise, ensure the attribute PICS code is NOT present in the PICS file."), - TestStep(4, "For every cluster present in the spec, for every client → server command in the cluster: If the cluster is present on the endpoint and the command id is present in the accepted commands list, ensure the PICS code for the accepted command is present in the PICS file. Otherwise, ensure the accepted command PICS code is not present in the PICS file."), - TestStep(5, "For every cluster present in the spec, for every server → client command in the cluster: If the cluster is present on the endpoint and the command id is present in the generated commands list, ensure the PICS code for the generated command is present in the PICS file. Otherwise, ensure the generated command PICS code is not present in the PICS file."), - TestStep(6, "For every cluster present in the spec, for every feature in the cluster: If the cluster is present on the endpoint and the feature is marked in the feature map, ensure the PICS code for the feature is present in the PICS file. Otherwise, ensure the feature PICS code is not present in the PICS file.")] + TestStep(2, "For every standard cluster: If the cluster is present on the endpoint, ensure the server-side PICS code for the cluster is present in the PICS file (e.g. OO.S for On/Off cluster).If the cluster is not present on the endpoint, ensure the cluster server PICS code is not present in the PICS file.", "PICS exactly match for server clusters."), + TestStep(3, "For every standard cluster, for every attribute in the cluster:If the cluster is present on the endpoint and the attribute ID is present in the AttributeList global attribute within the cluster, ensure the server-side PICS code for the attribute is present in the PICS file (e.g. OO.S.A000 for On/Off cluster’s OnOff attribute).Otherwise, ensure the attribute PICS code is NOT present in the PICS file.", "PICS exactly match for all attributes in all clusters."), + TestStep(4, "For every cluster present in the spec, for every client → server command in the cluster: If the cluster is present on the endpoint and the command id is present in the accepted commands list, ensure the PICS code for the accepted command is present in the PICS file. Otherwise, ensure the accepted command PICS code is not present in the PICS file.", "PICS exactly match for all accepted commands in all clusters."), + TestStep(5, "For every cluster present in the spec, for every server → client command in the cluster: If the cluster is present on the endpoint and the command id is present in the generated commands list, ensure the PICS code for the generated command is present in the PICS file. Otherwise, ensure the generated command PICS code is not present in the PICS file.", "PICS exactly match for all generated commands in all clusters."), + TestStep(6, "For every cluster present in the spec, for every feature in the cluster: If the cluster is present on the endpoint and the feature is marked in the feature map, ensure the PICS code for the feature is present in the PICS file. Otherwise, ensure the feature PICS code is not present in the PICS file.", "PICS exactly match for all features in all clusters."), + TestStep(7, "Ensure that the PICS_SDK_CI_ONLY PICS does not appear in the PICS file", "CI PICS is not present")] def test_TC_IDM_10_4(self): # wildcard read is done in setup_class @@ -175,6 +176,12 @@ def test_TC_IDM_10_4(self): location = ClusterPathLocation(endpoint_id=self.endpoint_id, cluster_id=cluster_id) self._check_and_record_errors(location, required, pics) + self.step(7) + if self.check_pics('PICS_SDK_CI_ONLY'): + self.record_error("PICS check", location=ProblemLocation(), + problem="PICS PICS_SDK_CI_ONLY found in PICS list. This PICS is disallowed for certification.") + self.success = False + if not self.success: self.fail_current_test("At least one PICS error was found for this endpoint") diff --git a/src/python_testing/TestMatterTestingSupport.py b/src/python_testing/TestMatterTestingSupport.py index eba9dc458383e5..d2a259f154adc6 100644 --- a/src/python_testing/TestMatterTestingSupport.py +++ b/src/python_testing/TestMatterTestingSupport.py @@ -625,7 +625,7 @@ def test_xml_pics(self): self.pics_assert('BINFO.S.A0010', True) self.pics_assert('BINFO.S.A0011', False) self.pics_assert('BINFO.S.A0012', True) - self.pics_assert('BINFO.S.A0013', True) + self.pics_assert('BINFO.S.A0013', False) self.pics_assert('BINFO.S.A0014', False) self.pics_assert('PICSDOESNOTEXIST', False) diff --git a/src/python_testing/matter_testing_support.py b/src/python_testing/matter_testing_support.py index 796397c57e0458..dbf4e14905015a 100644 --- a/src/python_testing/matter_testing_support.py +++ b/src/python_testing/matter_testing_support.py @@ -34,7 +34,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from enum import Enum -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple from chip.tlv import float32, uint @@ -417,7 +417,13 @@ class CustomCommissioningParameters: @dataclass -class AttributePathLocation: +class ProblemLocation: + def __str__(self): + return "UNKNOWN" + + +@dataclass +class AttributePathLocation(ProblemLocation): endpoint_id: int cluster_id: Optional[int] = None attribute_id: Optional[int] = None @@ -442,7 +448,7 @@ def __str__(self): @dataclass -class EventPathLocation: +class EventPathLocation(ProblemLocation): endpoint_id: int cluster_id: int event_id: int @@ -454,7 +460,7 @@ def __str__(self): @dataclass -class CommandPathLocation: +class CommandPathLocation(ProblemLocation): endpoint_id: int cluster_id: int command_id: int @@ -466,7 +472,7 @@ def __str__(self): @dataclass -class ClusterPathLocation: +class ClusterPathLocation(ProblemLocation): endpoint_id: int cluster_id: int @@ -476,7 +482,7 @@ def __str__(self): @dataclass -class FeaturePathLocation: +class FeaturePathLocation(ProblemLocation): endpoint_id: int cluster_id: int feature_code: str @@ -500,7 +506,7 @@ class ProblemSeverity(str, Enum): @dataclass class ProblemNotice: test_name: str - location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation] + location: ProblemLocation severity: ProblemSeverity problem: str spec_location: str = "" @@ -896,13 +902,13 @@ async def check_test_event_triggers_enabled(self): def print_step(self, stepnum: typing.Union[int, str], title: str) -> None: logging.info(f'***** Test Step {stepnum} : {title}') - def record_error(self, test_name: str, location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation], problem: str, spec_location: str = ""): + def record_error(self, test_name: str, location: ProblemLocation, problem: str, spec_location: str = ""): self.problems.append(ProblemNotice(test_name, location, ProblemSeverity.ERROR, problem, spec_location)) - def record_warning(self, test_name: str, location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation], problem: str, spec_location: str = ""): + def record_warning(self, test_name: str, location: ProblemLocation, problem: str, spec_location: str = ""): self.problems.append(ProblemNotice(test_name, location, ProblemSeverity.WARNING, problem, spec_location)) - def record_note(self, test_name: str, location: Union[AttributePathLocation, EventPathLocation, CommandPathLocation, ClusterPathLocation, FeaturePathLocation], problem: str, spec_location: str = ""): + def record_note(self, test_name: str, location: ProblemLocation, problem: str, spec_location: str = ""): self.problems.append(ProblemNotice(test_name, location, ProblemSeverity.NOTE, problem, spec_location)) def on_fail(self, record): diff --git a/src/python_testing/test_testing/example_pics_xml_basic_info.xml b/src/python_testing/test_testing/example_pics_xml_basic_info.xml index 3a8e279f3aa345..96f1a34f3e9557 100644 --- a/src/python_testing/test_testing/example_pics_xml_basic_info.xml +++ b/src/python_testing/test_testing/example_pics_xml_basic_info.xml @@ -182,7 +182,7 @@ Draft Does the DUT(server) support the CapabilityMinima attribute? 9.2.1. Attributes - index.html[pdf] M - true + false BINFO.S.A0014 diff --git a/src/python_testing/test_testing/test_IDM_10_4.py b/src/python_testing/test_testing/test_IDM_10_4.py new file mode 100644 index 00000000000000..ccf713450e2348 --- /dev/null +++ b/src/python_testing/test_testing/test_IDM_10_4.py @@ -0,0 +1,126 @@ +#!/usr/bin/env -S python3 -B +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys + +import chip.clusters as Clusters +from chip.clusters import Attribute + +try: + from pics_support import parse_pics_xml +except ImportError: + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + from pics_support import parse_pics_xml + +from MockTestRunner import MockTestRunner + +# Reachable attribute is off in the pics file +# MaxPathsPerInvoke is not include in the pics file +# Vendor ID is included on ON in the PICS file + + +def create_read(include_reachable: bool = False, include_max_paths: bool = False, include_vendor_id: bool = True) -> Attribute.AsyncReadTransaction.ReadResponse: + # Attribute read here is set to match the example_pics_xml_basic_info.xml in this directory + bi = Clusters.BasicInformation.Attributes + attrs = {bi.DataModelRevision: 1, + bi.VendorName: 'testVendor', + bi.ProductName: 'testProduct', + bi.ProductID: 0x8000, + bi.NodeLabel: 'label', + bi.Location: 'XX', + bi.HardwareVersion: 1, + bi.HardwareVersionString: 'one', + bi.SoftwareVersion: 2, + bi.SoftwareVersionString: 'two', + bi.ManufacturingDate: 'today', + bi.PartNumber: 'three', + bi.ProductURL: 'example.com', + bi.ProductLabel: 'myProduct', + bi.SerialNumber: 'ABCD1234', + bi.LocalConfigDisabled: False, + bi.UniqueID: 'Hashy-McHashface'} + if include_reachable: + attrs[bi.Reachable] = True + if include_max_paths: + attrs[bi.MaxPathsPerInvoke] = 2 + if include_vendor_id: + attrs[bi.VendorID] = 0xFFF1 + + attrs[bi.AttributeList] = [a.attribute_id for a in attrs.keys()] + attrs[bi.AcceptedCommandList] = [] + attrs[bi.GeneratedCommandList] = [] + attrs[bi.FeatureMap] = 0 + + resp = Attribute.AsyncReadTransaction.ReadResponse({}, [], {}) + resp.attributes = {0: {Clusters.BasicInformation: attrs}} + + tlv_attrs = {a.attribute_id: value for a, value in attrs.items()} + resp.tlvAttributes = {0: {Clusters.BasicInformation.id: tlv_attrs}} + + return resp + + +def main(): + # TODO: add the same test for commands and features + script_dir = os.path.dirname(os.path.realpath(__file__)) + with open(f'{script_dir}/example_pics_xml_basic_info.xml') as f: + pics = parse_pics_xml(f.read()) + test_runner = MockTestRunner('TC_pics_checker.py', 'TC_PICS_Checker', 'test_TC_IDM_10_4', 0, pics) + failures = [] + + # Success, include vendor ID, which IS in the pics file, and neither of the incorrect ones + resp = create_read() + print(resp) + if not test_runner.run_test_with_mock_read(resp): + failures.append("Test case failure: Vendor ID included - expected pass") + + # Failure because Vendor ID is not included in the read, but included in the PICS + resp = create_read(include_vendor_id=False) + if test_runner.run_test_with_mock_read(resp): + failures.append("Test case failure: Vendor ID not included - expected failure") + + # Failure because Reachable is included in the read, but not in the PICS + resp = create_read(include_reachable=True) + if test_runner.run_test_with_mock_read(resp): + failures.append("Test case failure: Reachable included but not in PICS- expected failure") + + # Failure because MaxPathsPerInvoke is included in the read, but not in the PICS + resp = create_read(include_max_paths=True) + if test_runner.run_test_with_mock_read(resp): + failures.append("Test case failure: MaxPathsPerInvoke included but not in PICS- expected failure") + + pics['PICS_SDK_CI_ONLY'] = True + test_runner.config.pics = pics + # This is a success case for the attributes (as seen above), but the test should fail because the CI PICS is added + resp = create_read() + if test_runner.run_test_with_mock_read(resp): + failures.append("Test case failure: SDK CI PICS is included - expected failure") + + test_runner.Shutdown() + + print( + f"Test of tests: PICS - test response incorrect: {len(failures)}") + for f in failures: + print(f) + + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())