diff --git a/.github/.wordlist.txt b/.github/.wordlist.txt index a6d2183cfd5ea1..c2820eee66d529 100644 --- a/.github/.wordlist.txt +++ b/.github/.wordlist.txt @@ -95,6 +95,7 @@ ASR AssertionError AST ASYNC +ATLs atomics att attId @@ -160,6 +161,7 @@ blockquote bluetoothd bluez BOOL +booleans BooleanState bootable Bootloader @@ -380,6 +382,7 @@ DefaultOTARequestor DefaultOTARequestorDriver DefaultOTARequestorStorage DefaultSuccess +defaultValue definedValue DehumidificationControl DelayedActionTime @@ -503,6 +506,7 @@ emberAfExternalAttributeReadCallback emberAfExternalAttributeWriteCallback EmberAfInitializeAttributes emberAfSetDynamicEndpoint +emsp EnableNetwork EnableWiFiNetwork endian @@ -604,6 +608,7 @@ GenericWiFiConfigurationManagerImpl GetDeviceId GetDeviceInfo GetDns +GetInDevelopmentTests GetIP getManualTests GetSafeAttributePersistenceProvider @@ -673,6 +678,7 @@ IasWd iaszone ibb ICA +ICAC ICD ICDs iCloud @@ -724,6 +730,7 @@ IoT ipaddr iPadOS ipadr +IPK ipp iptables iputils @@ -935,6 +942,7 @@ namespacing nano natively navpad +nbsp NCP ncs nding @@ -1076,6 +1084,7 @@ Pigweed PinCode pinrequest PIXIT +PIXITs pkgconfig PKI plaintext @@ -1155,6 +1164,7 @@ RADVD raspberryPi RasPi rAv +RCAC RCP ReadAttribute ReadConfigValue @@ -1219,6 +1229,7 @@ RTOS RTT RTX runArgs +runIf RUNAS RunMain runtime @@ -1227,6 +1238,7 @@ rw RXD sandboxed saveAs +saveDataVersschemaionAs sbin scalability scalable @@ -1377,6 +1389,7 @@ TestArray TestCluster TestConstraints TestEmptyString +TestEqualities TestGenExample TestGroupDemoConfig TestMultiRead diff --git a/.spellcheck.yml b/.spellcheck.yml index e3e470a696bbeb..c04880827addba 100644 --- a/.spellcheck.yml +++ b/.spellcheck.yml @@ -20,7 +20,7 @@ # # Actual run: # -# pyspelling pyspelling --config .spellcheck.yml +# pyspelling --config .spellcheck.yml matrix: - name: markdown @@ -65,6 +65,6 @@ matrix: # converts markdown to HTML - pyspelling.filters.markdown: sources: - - '**/*.md|!third_party/**|!examples/common/**/repo/**|!docs/ERROR_CODES.md|!docs/clusters.md' + - '**/*.md|!third_party/**|!examples/common/**/repo/**|!docs/ERROR_CODES.md|!docs/clusters.md|!docs/testing/yaml_schema.md|!docs/testing/yaml_pseudocluster.md' aspell: ignore-case: true diff --git a/docs/index.md b/docs/index.md index 12218445600743..8447850784787f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,6 +16,7 @@ cluster_and_device_type_dev/index guides/index style/index examples/index +testing/index tools/index BUG_REPORT code_generation diff --git a/docs/testing/ci_testing.md b/docs/testing/ci_testing.md new file mode 100644 index 00000000000000..7f974bb6f03604 --- /dev/null +++ b/docs/testing/ci_testing.md @@ -0,0 +1,6 @@ +# CI testing + +This file is a placeholder for information on how to run tests in the CI. + +NOTE: discuss in particular triggers direct to the device, test event triggers +and the CI pics. diff --git a/docs/testing/index.md b/docs/testing/index.md new file mode 100644 index 00000000000000..0b9bbf4c494a38 --- /dev/null +++ b/docs/testing/index.md @@ -0,0 +1,31 @@ +# Testing Guides + +The following guide provide an introduction to the testing mechanisms available +in the SDK. + +```{toctree} +:glob: +:maxdepth: 1 +:hidden: + +* +``` + +## Unit testing + +- [Unit tests](./unit_testing.md) + +## Integration and Certification tests + +- [Integration and Certification tests](./integration_tests.md) +- [YAML](./yaml.md) +- [Python testing framework](./python.md) +- [Enabling tests in the CI](./ci_testing.md) + +## PICS and PIXIT + +- [PICS and PIXIT](./pics_and_pixit.md) + +## Testing in the CI + +- [CI testing](./ci_testing.md) diff --git a/docs/testing/integration_tests.md b/docs/testing/integration_tests.md new file mode 100644 index 00000000000000..46a20ebb8ac090 --- /dev/null +++ b/docs/testing/integration_tests.md @@ -0,0 +1,46 @@ +# Integration and Certification Tests + +Integration tests use a server and a controller or controllers to test the +behavior of a device. Certification tests are all integration tests. For +certified products, the device under test (DUT) is tested against one of the SDK +controller implementations (either chip-tool or the python-based controller, +depending on the test type). For software component certification, the software +component is tested against a sample device built from the SDK. + +Certification tests require an accompanying certification test plan in order to +be used in the certification testing process. More information about test plans +can be found in the +[test plans repository](https://github.com/CHIP-Specifications/chip-test-plans/tree/master/docs). +Integration testing can also be used outside of the certification testing +program to test device behavior in the SDK. Certification tests are all run in +the [CI](./ci_testing). + +There are two main integration test types: + +- [YAML](./yaml.md) +- [Python framework](./python.md) + +YAML is a human-readable serialization language that uses structured tags to +define test steps. Tests are defined in YAML, and parsed and run through a +runner that is backed by the chip-tool controller. + +The Python framework tests are written in python and use the +[Mobly](https://github.com/google/mobly) test framework to execute tests. + +## Which test framework to use + +Both types of tests can be run through the Test Harness for certification +testing, locally for the purposes of development and in the CI for the SDK. The +appropriate test framework to use is whatever lets you automate your tests in a +way that is understandable, readable, and has the features you need + +- YAML + - pros: more readable, simpler to write, easy for ATLs to parse and + understand + - cons: conditionals are harder (not all supported), no branch control, + schema not well documented +- python + - pros: full programming language, full control API with support for core + (certs, commissioning, etc), less plumbing if you need to add features, + can use python libraries + - cons: more complex, can be harder to read diff --git a/docs/testing/pics_and_pixit.md b/docs/testing/pics_and_pixit.md new file mode 100644 index 00000000000000..ee8901c66fecab --- /dev/null +++ b/docs/testing/pics_and_pixit.md @@ -0,0 +1,10 @@ +# PICS and PIXITs + +Placeholder file for PICS and PIXIT info + +- PICS formats - XML vs. test harness +- PICS in CI +- PICS tool and how we practically use it in Matter +- PICS checker test +- PIXITs in tests and how to set them +- Why you should avoid using both of these things. diff --git a/docs/testing/python.md b/docs/testing/python.md new file mode 100644 index 00000000000000..3af81e2394fe85 --- /dev/null +++ b/docs/testing/python.md @@ -0,0 +1,6 @@ +# Python framework tests + +This file is a placeholder for python framework test information. + +NOTE: be sure to include information about how you need to commission with the +python controller, not chip-tool and how to do that in the scripts diff --git a/docs/testing/unit_testing.md b/docs/testing/unit_testing.md new file mode 100644 index 00000000000000..e62940f15a9c7c --- /dev/null +++ b/docs/testing/unit_testing.md @@ -0,0 +1,3 @@ +# Unit testing + +This doc is a placeholder for an guide around unit tests. diff --git a/docs/testing/yaml.md b/docs/testing/yaml.md new file mode 100644 index 00000000000000..2099d623c1b3db --- /dev/null +++ b/docs/testing/yaml.md @@ -0,0 +1,362 @@ +# YAML tests + +YAML is a structured, human-readable data-serialization language. Much like json +or proto, YAML refers to the structure and parser, and the schema used for any +particular application is defined by the application. + +In Matter, we use YAML for describing tests and test steps. A YAML parser and +runner is then used to translate the YAML instructions into actions used to +interact with the device under test (DUT). + +The main runner we use for testing in Matter parses the YAML instructions into +chip-tool commands. + +The schema description for the Matter test YAML is available here: +[YAML Schema](./yaml_schema.md) + +## Writing YAML tests + +Most YAML tests are written for certification. These follow a standard format +that is used to display the test easily in the test harness. + +### Placeholder for anatomy of a yaml test - need diagram + +### Placeholder for anatomy of a test step - need diagram + +### Common actions + +#### Sending a cluster command + +The following shows a test step sending a simple command with no arguments. + +``` + - label: "This label gets printed" + cluster: "On/Off" + command: "On" +``` + +- label - label to print before performing the test step +- cluster - name of the cluster to send the command to +- command - name of the command to send + +This send the On command to the On/Off cluster on the DUT. For most tests, the +nodeID of the DUT and endpoint for the cluster are defined in the top-level +config section of the file and applied to every test step. However, these can +also be overwritten in the individual test steps. + +The following shows how to send a command with arguments: + +``` + - label: "This label gets printed before the test step" + command: "MoveToColor" + arguments: + values: + - name: "ColorX" + value: 32768 + - name: "ColorY" + value: 19660 + - name: "TransitionTime" + value: 0 + - name: "OptionsMask" + value: 0 + - name: "OptionsOverride" + value: 0 +``` + +- label - label to print before performing the test step +- command - name of the command to send +- argument - this is a list parameter that takes either a "value" or "values" + tag. Commands with arguments all use structured fields, which require the + "values" tag with a list. Each of the fields is represented by a "name" and + "value" pair + +In this command, the cluster: tag is elided. The cluster for the entire test can +be set in the config section at the top of the test. This can be overwritten for +individual test steps (as above). + +#### Reading and writing attributes + +Reading and writing attributes is represented in the Matter test YAML schemas as +a special command that requires an additional "attribute" tag. + +The following YAML would appear as a test step, and shows how to read an +attribute. + +``` +- label: "TH reads the ClusterRevision from DUT" + command: "readAttribute" + attribute: "ClusterRevision" +``` + +The following YAML would appear as a test step and shows how to write an +attribute. Commands to write attributes always require an argument: tag. + +``` +- label: "Write example attribute" + command: "writeAttribute" + attribute: "ExampleAttribute" + arguments: + value: 1 +``` + +#### Parsing Responses + +After sending a command or read or write attribute request, you may want to +verify the response. This is done using the "response" tag with various +sub-tags. + +The following shows a simple response parsing with two (somewhat redundant) +checks. + +``` +- label: "TH reads the ClusterRevision from DUT" + command: "readAttribute" + attribute: "ClusterRevision" + response: + value: 1 + constraints: + minValue: 1 +``` + +The following tags can be used to parse the response + +| Example | Description | +| :---------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------- | +| response:
 value: [1, 2, 3, 4] | must match exactly. Variables and saveAs values allowed | +| response:
 values:
  - name: response_field
     value: 1 | Must match exactly
Use for commands that return command responses with named fields | +| response:
  error: CONSTRAINT_ERROR | expect an error back (omit for success)
Variables and saveAs values will not work. | +| response:
 constraints: | more complex checks - see [Schema](./yaml_schema.md) for a complete description | + +#### Lists and structs + +Lists and structs can be represented as follows: + +Lists: `[1,2,3,4,5]` + +structs: `{field1:value, field2:value}` + +lists of structs: + +``` +[ + +{field1:value, field2:value, optionalfield:value}, + +{field1:value, field2:value}, + +] +``` + +Note that structs are different than command and command response fields, which +are represented using name:, value: tags. + +#### Pseudo clusters + +Tests often require functionality that is not strictly cluster-based. Some of +this functionality is supported in YAML using pseudo-clusters. These clusters +accept command: tags like the DUT clusters to control the pseudo-cluster +functionality. + +Some of the more common functionality is shown below: + +Establishing a connection to the DUT. This is the first step in nearly every +test. + +``` + - label: "Establish a connection to the DUT" + cluster: "DelayCommands" + command: "WaitForCommissionee" + arguments: + values: + - name: "nodeId" + value: nodeId +``` + +Wait for a user action: + +``` + - label: "Do a simple user prompt message. Expect 'y' to pass." + cluster: "LogCommands" + command: "UserPrompt" + arguments: + values: + - name: "message" + value: "Please enter 'y' for success" + - name: "expectedValue" + value: "y" +``` + +Wait for a time: + +``` + - label: "Wait for 5S" + cluster: "DelayCommands" + command: "WaitForMs" + arguments: + values: + - name: "ms" + value: 5000 +``` + +A full description of the available pseudo-clusters and their commands is +available at [Pseudo-cluster description](./yaml_pseudocluster.md). + +#### Config variables and saveAs: + +Certain tags can use variables that are either declared in the config: section +or saved from other steps. Variables that are declared in the config can be +overwritten on the command line when running locally or through the config file +in the test harness. + +To declare config variables in the config section, use a label with the desired +name, then provide the type and defaultValue tags as sub-tags. + +``` +config: + nodeId: 0x12344321 + cluster: "Unit Testing" + endpoint: 1 + myArg1: + type: int8u + defaultValue: 5 +``` + +Variables can also be saved from responses: + +``` + - label: "Send Test Add Arguments Command" + command: "TestAddArguments" + arguments: + values: + - name: "arg1" + value: 3 + - name: "arg2" + value: 17 + response: + values: + - name: "returnValue" + saveAs: TestAddArgumentDefaultValue + value: 20 +``` + +Variables can then be used in later steps: + +``` + - label: "Send Test Add Arguments Command" + command: "TestAddArguments" + arguments: + values: + - name: "arg1" + value: 3 + - name: "arg2" + value: 17 + response: + values: + - name: "returnValue" + value: TestAddArgumentDefaultValue +``` + +Tags where variables can be used are noted in the +[schema description](./yaml_schema.md). + +Config variables can be used to implement PIXIT values in tests. + +#### Gating tests and steps: PICS, TestEqualities and runIf + +The PICS tag can be used to unconditionally gate a test step on the PICS value +in the file. + +The PICS tag can handle standard boolean operations on pics (!, ||, &&, ()). + +A PICS tag at the top level of the file can be used to gate the entire test in +the test harness. Note that full-test gating is not currently implemented in the +local runner or in the CI. + +Some test steps need to be gated on values from earlier in the test. In these +cases, PICS cannot be used. Instead, the runIf: tag can be used. This tag +requires a boolean value. To convert values to booleans, the TestEqualities +function can be use. See +[TestEqualities](https://github.com/project-chip/connectedhomeip/blob/master/src/app/tests/suites/TestEqualities.yaml) +for an example of how to use this pseudo-cluster. + +## Running YAML tests + +YAML scripts are parsed and run using a python-based runner program that parses +the file, then translates the tags into chip-tool commands, and sends those +commands over a socket to chip-tool (running in interactive mode). + +### Running locally + +#### Commissioning the DUT + +All YAML tests assume that the DUT has previously been commissioned before +running. DUTs should be commissioned using chip-tool. Use the same KVS file when +running the test. + +#### Running the tests + +There are several options for running tests locally. Because the YAML runner +uses python, it is necessary to compile and install the chip python package +before using any YAML runner script. + +``` +./scripts/build_python.sh -i py +source py/bin/activate +``` + +Compile chip-tool: + +``` +./scripts/build/build_examples.py --target linux-x64-chip-tool build + +``` + +NOTE: use the target appropriate to your system + +[chiptool.py](https://github.com/project-chip/connectedhomeip/blob/master/scripts/tests/yaml/chiptool.py) +can be used to run tests against a commissioned DUT (commissioned by chip-tool). +This will start an interactive instance of chip-tool automatically. + +``` +./scripts/tests/yaml/chiptool.py tests Test_TC_OO_2_1 --server_path ./out/linux-x64-chip-tool/chip-tool + +``` + +NOTE: substitute the appropriate test name and chip-tool path as appropriate. + +A list of available tests can be generated using: + +``` +./scripts/tests/yaml/chiptool.py list +``` + +Config variables can be passed to chiptool.py after the script by separating +with -- + +``` +./scripts/tests/yaml/chiptool.py tests Test_TC_OO_2_1 --server_path ./out/linux-x64-chip-tool/chip-tool -- nodeId 0x12344321 + +``` + +#### Factory resetting the DUT + +On the host machine, you can simulate a factory reset by deleting the KVS file. +If you did not specify a location for the KVS file when starting the +application, the KVS file will be in /tmp as chip_kvs + +### Running in the CI + +- YAML tests added to the certification directory get run automatically + - src/app/tests/suites/certification/ + - PICS file: src/app/tests/suites/certification/ci-pics-values +- If you DON’T want to run a test in the CI + - (ex under development), add it to \_GetInDevelopmentTests in + `scripts/tests/chiptest/__init__.py` + +Please see [CI testing](./ci_testing.md) for more information about how to set +up examples apps, PICS and PIXIT values for use in the CI. + +### Running in the TH + +TODO: Do we have a permanent link to the most up to date TH documentation? If +so, add here. diff --git a/docs/testing/yaml_pseudocluster.md b/docs/testing/yaml_pseudocluster.md new file mode 100644 index 00000000000000..d6296e986d7934 --- /dev/null +++ b/docs/testing/yaml_pseudocluster.md @@ -0,0 +1,50 @@ + + +# YAML Pseudo-clusters + +CommissionerCommands |command|args|arg type| arg optional| |:---|:---|:---|:---| +|PairWithCode|nodeId
payload
discoverOnce|node_id
char_string
boolean|false
false
true| +|Unpair|nodeId|node_id|false| |GetCommissionerNodeId|||| +|GetCommissionerNodeIdResponse|nodeId|node_id|false| +|GetCommissionerRootCertificate|||| +|GetCommissionerRootCertificateResponse|RCAC|OCTET_STRING|false| +|IssueNocChain|Elements
nodeId|octet_string
node_id|false
false| +|IssueNocChainResponse|NOC
ICAC
RCAC
IPK|octet_string
octet_string
octet_string
octet_string|false
false
false
false| + +DelayCommands |command|args|arg type| arg optional| |:---|:---|:---|:---| +|WaitForCommissioning|||| +|WaitForCommissionee|nodeId
expireExistingSession|node_id
bool|false
true| +|WaitForMs|ms|int16u|false| +|WaitForMessage|registerKey
message|char_string
char_string|false
false| + +DiscoveryCommands |command|args|arg type| arg optional| |:---|:---|:---|:---| +|FindCommissionable|||| +|FindCommissionableByShortDiscriminator|value|int16u|false| +|FindCommissionableByLongDiscriminator|value|int16u|false| +|FindCommissionableByCommissioningMode|||| +|FindCommissionableByVendorId|value|vendor_id|false| +|FindCommissionableByDeviceType|value|devtype_id|false| |FindCommissioner|||| +|FindCommissionerByVendorId|value|vendor_id|false| +|FindCommissionerByDeviceType|value|devtype_id|false| +|FindResponse|hostName
instanceName
longDiscriminator
shortDiscriminator
vendorId
productId
commissioningMode
deviceType
deviceName
rotatingId
rotatingIdLen
pairingHint
pairingInstruction
supportsTcp
numIPs
port
mrpRetryIntervalIdle
mrpRetryIntervalActive
mrpRetryActiveThreshold
isICDOperatingAsLIT|char_string
char_string
int16u
int16u
vendor_id
int16u
int8u
devtype_id
char_string
octet_string
int64u
int16u
char_string
boolean
int8u
int16u
int32u
int32u
int16u
boolean|false
false
false
false
false
false
false
false
false
false
false
false
false
false
false
false
true
true
true
true| + +EqualityCommands |command|args|arg type| arg optional| |:---|:---|:---|:---| +|BooleanEquals|Value1
Value2|boolean
boolean|false
false| +|SignedNumberEquals|Value1
Value2|int64s
int64s|false
false| +|UnsignedNumberEquals|Value1
Value2|int64u
int64u|false
false| +|EqualityResponse|Equals|bool|false| + +LogCommands |command|args|arg type| arg optional| |:---|:---|:---|:---| +|Log|message|char_string|false| +|UserPrompt|message
expectedValue|char_string
char_string|false
true| + +SystemCommands |command|args|arg type| arg optional| |:---|:---|:---|:---| +|Start|registerKey
discriminator
port
minCommissioningTimeout
kvs
filepath
otaDownloadPath|char_string
int16u
int16u
int16u
char_string
char_string
char_string|true
true
true
true
true
true
true| +|Stop|registerKey|char_string|true| |Reboot|registerKey|char_string|true| +|FactoryReset|registerKey|char_string|true| +|CreateOtaImage|otaImageFilePath
rawImageFilePath
rawImageContent|char_string
char_string
char_string|false
false
false| +|CompareFiles|file1
file2|char_string
char_string|false
false| diff --git a/docs/testing/yaml_schema.md b/docs/testing/yaml_schema.md new file mode 100644 index 00000000000000..9a3ee96fa2959b --- /dev/null +++ b/docs/testing/yaml_schema.md @@ -0,0 +1,35 @@ + + +# YAML Schema + +YAML schema |key | type| supports variables |:---|:---|:---| |name |str|| |PICS +|str,list|| |config | | | |  nodeId |int|| |  cluster |str|| |  +endpoint |int|| |  _variableName_ | | | |    type |type|| |  +  defaultValue |Any|| |tests | | | |  label |str|| |  identity +|str|| |  nodeId |int|Y| |  runIf |str|| |  groupId |int|Y| +|  endpoint |int|Y| |  cluster |str|| |  attribute |str|| |  +command |str|| |  event |str|| |  eventNumber |int|Y| |  disabled +|bool|| |  fabricFiltered |bool|| |  verification |str|| |  PICS +|str|| |  arguments | | | |    values | | | |      +value |NoneType,bool,int,float,dict,list|Y| |      name |str|| +|    value |NoneType,bool,int,float,dict,list|Y| |  response | |Y +| |    value |NoneType,bool,int,float,dict,list|Y| |    name +|str|| |    error |str|| |    clusterError |int|| |  +  constraints | | | |      hasValue |bool|| |    +  type |str|| |      minLength |int|| |      +maxLength |int|| |      isHexString |bool|| |      +startsWith |str|| |      endsWith |str|| |      +isUpperCase |bool|| |      isLowerCase |bool|| |    +  minValue |int,float|Y| |      maxValue |int,float|Y| +|      contains |list|| |      excludes |list|| +|      hasMasksSet |list|| |      hasMasksClear +|list|| |      notValue |NoneType,bool,int,float,list,dict|Y| +|      anyOf |list|| |    saveAs |str|| |    +saveDataVersschemaionAs |str|| |  saveResponseAs |str|| |  minInterval +|int|| |  maxInterval |int|| |  keepSubscriptions |bool|| |  +timeout |int|| |  timedInteractionTimeoutMs |int|| |  dataVersion +|list,int|Y| |  busyWaitMs |int|| |  wait |str|| diff --git a/scripts/py_matter_yamltests/generate_pseudo_cluster_doc_tables.py b/scripts/py_matter_yamltests/generate_pseudo_cluster_doc_tables.py new file mode 100644 index 00000000000000..11a7b2b9dc9034 --- /dev/null +++ b/scripts/py_matter_yamltests/generate_pseudo_cluster_doc_tables.py @@ -0,0 +1,56 @@ +# Copyright (c) 2024 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import xml.etree.ElementTree as ElementTree + +from matter_yamltests.pseudo_clusters.pseudo_clusters import get_default_pseudo_clusters + +SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) +WARNING = ("\n\n") + + +def create_tables(): + pseudo_clusters = get_default_pseudo_clusters() + + doc_path = os.path.abspath(os.path.join( + SCRIPT_DIR, '..', '..', 'docs', 'testing', 'yaml_pseudocluster.md')) + with open(doc_path, "w") as f: + f.writelines(WARNING) + f.writelines('# YAML Pseudo-clusters\n\n') + + for cluster in pseudo_clusters.clusters: + f.writelines(f'\n\n{cluster.name}\n') + f.writelines('|command|args|arg type| arg optional|\n') + f.writelines('|:---|:---|:---|:---|\n') + + et = ElementTree.fromstring(cluster.definition) + cluster_xml = next(et.iter('cluster')) + for command_xml in cluster_xml.iter('command'): + cmd = command_xml.get('name') + arg = '
'.join([arg_xml.get('name') + for arg_xml in command_xml.iter('arg')]) + argtype = '
'.join([arg_xml.get('type') + for arg_xml in command_xml.iter('arg')]) + optional = '
'.join([arg_xml.get('optional', 'false') + for arg_xml in command_xml.iter('arg')]) + + f.writelines(f'|{cmd}|{arg}|{argtype}|{optional}|\n') + + +create_tables() diff --git a/scripts/py_matter_yamltests/generate_yaml_doc_tables.py b/scripts/py_matter_yamltests/generate_yaml_doc_tables.py new file mode 100644 index 00000000000000..4deb3abe00c09c --- /dev/null +++ b/scripts/py_matter_yamltests/generate_yaml_doc_tables.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from typing import TextIO + +from matter_yamltests.yaml_loader import SchemaTree, yaml_tree + +SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) +WARNING = ("\n\n") + + +def get_type_list_and_vars(typetuple) -> (list[type], bool): + # If str is one of the supported types, and other base types are supported, + # this means it supports variables. + # This is a heuristic, but it's true for now. + try: + typelist = list(typetuple) + if str in typelist: + reduced = [t for t in typelist if t != str] + if reduced != [list]: + return (reduced, True) + return (typelist, False) + except TypeError: + return ([typetuple], False) + + +def print_tree(f: TextIO, indent: str, tree: SchemaTree) -> None: + for tag, typetuple in tree.schema.items(): + vars_str = "" + typelist, vars = get_type_list_and_vars(typetuple) + vars_str = "Y" if vars else "" + typestr = ','.join([t.__name__ for t in typelist]) + + try: + child = tree.children[tag] + f.writelines([f'|{indent}{tag} | |{vars_str} |\n']) + print_tree(f, indent+'  ', child) + except (TypeError, KeyError): + f.writelines([f'|{indent}{tag} |{typestr}|{vars_str}|\n']) + + +def print_table(title: str, tree: SchemaTree) -> None: + doc_path = os.path.abspath(os.path.join( + SCRIPT_DIR, '..', '..', 'docs', 'testing', 'yaml_schema.md')) + with open(doc_path, "w") as f: + f.writelines(WARNING) + f.writelines('# YAML Schema\n\n') + f.writelines([f'{title}\n', + '|key | type| supports variables\n', + '|:---|:---|:---|\n']) + print_tree(f, '', tree) + + +print_table('YAML schema', yaml_tree) diff --git a/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py b/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py index 01b9e040f3efd8..eebd570256d1ac 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py +++ b/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Union +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Tuple, Union from .errors import (TestStepArgumentsValueError, TestStepError, TestStepGroupEndPointError, TestStepGroupResponseError, TestStepInvalidTypeError, TestStepKeyError, TestStepNodeIdAndGroupIdError, TestStepResponseVariableError, @@ -30,6 +33,121 @@ import yaml +_TOP_LEVEL_SCHEMA = { + 'name': str, + 'PICS': (str, list), + 'config': dict, + 'tests': list, +} + +_TEST_STEP_SCHEMA = { + 'label': str, + 'identity': str, + 'nodeId': (int, str), # Can be a variable. + 'runIf': str, # Should be a variable. + 'groupId': (int, str), # Can be a variable. + 'endpoint': (int, str), # Can be a variable + 'cluster': str, + 'attribute': str, + 'command': str, + 'event': str, + 'eventNumber': (int, str), # Can be a variable. + 'disabled': bool, + 'fabricFiltered': bool, + 'verification': str, + 'PICS': str, + 'arguments': dict, + 'response': (dict, list, str), # Can be a variable + 'saveResponseAs': str, + 'minInterval': int, + 'maxInterval': int, + 'keepSubscriptions': bool, + 'timeout': int, + 'timedInteractionTimeoutMs': int, + 'dataVersion': (list, int, str), # Can be a variable + 'busyWaitMs': int, + 'wait': str, +} + +_TEST_STEP_ARGUMENTS_SCHEMA = { + 'values': list, + 'value': (type(None), bool, str, int, float, dict, list), +} + +_TEST_STEP_ARGUMENTS_VALUES_SCHEMA = { + 'value': (type(None), bool, str, int, float, dict, list), + 'name': str, +} + +_TEST_STEP_RESPONSE_SCHEMA = { + 'value': (type(None), bool, str, int, float, dict, list), + 'name': str, + 'error': str, + 'clusterError': int, + 'constraints': dict, + 'saveAs': str, + 'saveDataVersschemaionAs': str, +} + +_TEST_STEP_RESPONSE_CONSTRAINTS_SCHEMA = { + 'hasValue': bool, + 'type': str, + 'minLength': int, + 'maxLength': int, + 'isHexString': bool, + 'startsWith': str, + 'endsWith': str, + 'isUpperCase': bool, + 'isLowerCase': bool, + 'minValue': (int, float, str), # Can be a variable + 'maxValue': (int, float, str), # Can be a variable + 'contains': list, + 'excludes': list, + 'hasMasksSet': list, + 'hasMasksClear': list, + 'notValue': (type(None), bool, str, int, float, list, dict), + 'anyOf': list +} + +# Note: this is not used in the loader, just provided for information in the schema tree +_CONFIG_SCHEMA = { + 'nodeId': int, + 'cluster': str, + 'endpoint': int, + '_variableName_': str, +} + +# Note: this is not used in the loader, just provided for information in the schema tree +_CONFIG_VARIABLE_SCHEMA = { + 'type': type, + 'defaultValue': Any, +} + + +@dataclass +class SchemaTree: + schema: dict[str, type] + children: Union[dict[str, SchemaTree], None] = None + + +_constraint_tree = SchemaTree(schema=_TEST_STEP_RESPONSE_CONSTRAINTS_SCHEMA) +_response_tree = SchemaTree(schema=_TEST_STEP_RESPONSE_SCHEMA, children={ + 'constraints': _constraint_tree}) + +_arguments_values_tree = SchemaTree(schema=_TEST_STEP_ARGUMENTS_VALUES_SCHEMA) +_arguments_tree = SchemaTree(schema=_TEST_STEP_ARGUMENTS_SCHEMA, children={ + 'values': _arguments_values_tree}) + +_test_step_tree = SchemaTree(schema=_TEST_STEP_SCHEMA, children={ + 'arguments': _arguments_tree, 'response': _response_tree}) + +_config_variable_tree = SchemaTree(schema=_CONFIG_VARIABLE_SCHEMA) +_config_tree = SchemaTree(schema=_CONFIG_SCHEMA, children={ + '_variableName_': _config_variable_tree}) + +yaml_tree = SchemaTree(schema=_TOP_LEVEL_SCHEMA, children={ + 'tests': _test_step_tree, 'config': _config_tree}) + class YamlLoader: """This class loads a file from the disk and validates that the content is a well formed yaml test.""" @@ -58,12 +176,7 @@ def load(self, yaml_file: str) -> Tuple[str, Union[list, str], dict, list]: return (filename, name, pics, config, tests) def __check_content(self, content): - schema = { - 'name': str, - 'PICS': (str, list), - 'config': dict, - 'tests': list, - } + schema = _TOP_LEVEL_SCHEMA try: self.__check(content, schema) @@ -86,34 +199,7 @@ def __check_content(self, content): raise def __check_test_step(self, config: dict, content): - schema = { - 'label': str, - 'identity': str, - 'nodeId': (int, str), # Can be a variable. - 'runIf': str, # Should be a variable. - 'groupId': (int, str), # Can be a variable. - 'endpoint': (int, str), # Can be a variable - 'cluster': str, - 'attribute': str, - 'command': str, - 'event': str, - 'eventNumber': (int, str), # Can be a variable. - 'disabled': bool, - 'fabricFiltered': bool, - 'verification': str, - 'PICS': str, - 'arguments': dict, - 'response': (dict, list, str), # Can be a variable - 'saveResponseAs': str, - 'minInterval': int, - 'maxInterval': int, - 'keepSubscriptions': bool, - 'timeout': int, - 'timedInteractionTimeoutMs': int, - 'dataVersion': (list, int, str), # Can be a variable - 'busyWaitMs': int, - 'wait': str, - } + schema = _TEST_STEP_SCHEMA self.__check(content, schema) self.__rule_node_id_and_group_id_are_mutually_exclusive(content) @@ -145,10 +231,7 @@ def __check_test_step(self, config: dict, content): self.__check_test_step_response(response) def __check_test_step_arguments(self, content): - schema = { - 'values': list, - 'value': (type(None), bool, str, int, float, dict, list), - } + schema = _TEST_STEP_ARGUMENTS_SCHEMA self.__check(content, schema) @@ -158,10 +241,7 @@ def __check_test_step_arguments(self, content): [self.__check_test_step_argument_value(x) for x in values] def __check_test_step_argument_value(self, content): - schema = { - 'value': (type(None), bool, str, int, float, dict, list), - 'name': str, - } + schema = _TEST_STEP_ARGUMENTS_VALUES_SCHEMA self.__check(content, schema) @@ -177,15 +257,7 @@ def __check_test_step_response(self, content): self.__check_test_step_response_value(content) def __check_test_step_response_value(self, content, allow_name_key=False): - schema = { - 'value': (type(None), bool, str, int, float, dict, list), - 'name': str, - 'error': str, - 'clusterError': int, - 'constraints': dict, - 'saveAs': str, - 'saveDataVersionAs': str, - } + schema = _TEST_STEP_RESPONSE_SCHEMA if allow_name_key: schema['name'] = str @@ -197,25 +269,7 @@ def __check_test_step_response_value(self, content, allow_name_key=False): self.__check_test_step_response_value_constraints(constraints) def __check_test_step_response_value_constraints(self, content): - schema = { - 'hasValue': bool, - 'type': str, - 'minLength': int, - 'maxLength': int, - 'isHexString': bool, - 'startsWith': str, - 'endsWith': str, - 'isUpperCase': bool, - 'isLowerCase': bool, - 'minValue': (int, float, str), # Can be a variable - 'maxValue': (int, float, str), # Can be a variable - 'contains': list, - 'excludes': list, - 'hasMasksSet': list, - 'hasMasksClear': list, - 'notValue': (type(None), bool, str, int, float, list, dict), - 'anyOf': list - } + schema = _TEST_STEP_RESPONSE_CONSTRAINTS_SCHEMA self.__check(content, schema)