Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a way to have tests run only in the python-chip-tool driver. #28526

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions examples/chip-tool/templates/tests/commands.zapt
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,18 @@ public:
}
};

class PythonRunnerOnlyTestList : public Command
{
public:
PythonRunnerOnlyTestList() : Command("list-python-runner-only") {};
CHIP_ERROR Run() override
{
{{#chip_tests "../../../../src/app/tests/suites/pythonRunnerOnlyTests.json" includeAllClusters=true}}printf("{{filename}}\n");{{/chip_tests}}

return CHIP_NO_ERROR;
}
};

{{>test_cluster tests="../../../../src/app/tests/suites/ciTests.json" credsIssuerConfigArg=true needsWaitDuration=true}}
{{>test_cluster tests="../../../../src/app/tests/suites/manualTests.json" credsIssuerConfigArg=true needsWaitDuration=true}}

Expand All @@ -49,6 +61,7 @@ void registerCommandsTests(Commands & commands, CredentialIssuerCommands * creds
#if CONFIG_ENABLE_YAML_TESTS
make_unique<TestList>(),
make_unique<ManualTestList>(),
make_unique<PythonRunnerOnlyTestList>(),
{{#chip_tests "../../../../src/app/tests/suites/ciTests.json" includeAllClusters=true}}
make_unique<{{filename}}Suite>(credsIssuerConfig),
{{/chip_tests}}
Expand Down
4 changes: 2 additions & 2 deletions scripts/setup/zap.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@
"mac-amd64",
"windows-amd64"
],
"tags": ["version:[email protected].01-nightly.1"]
"tags": ["version:[email protected].04-nightly.1"]
},
{
"_comment": "Always get the amd64 version on mac until usable arm64 zap build is available",
"path": "fuchsia/third_party/zap/mac-amd64",
"platforms": ["mac-arm64"],
"tags": ["version:[email protected].01-nightly.1"]
"tags": ["version:[email protected].04-nightly.1"]
}
]
}
2 changes: 1 addition & 1 deletion scripts/setup/zap.version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v2023.08.01-nightly
v2023.08.04-nightly
54 changes: 43 additions & 11 deletions scripts/tests/chiptest/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,27 +129,48 @@ def _GetSlowTests() -> Set[str]:

def _GetInDevelopmentTests() -> Set[str]:
"""Tests that fail in YAML for some reason."""
return {
"Test_TC_TIMESYNC_1_1.yaml", # Time sync SDK is not yet ready
"Test_TC_TIMESYNC_2_3.yaml", # Time sync SDK is not yet ready
"Test_TC_PSCFG_1_1.yaml", # Power source configuration cluster is deprecated and removed from all-clusters
"Test_TC_PSCFG_2_1.yaml", # Power source configuration cluster is deprecated and removed from all-clusters
"Test_TC_PSCFG_2_2.yaml", # Power source configuration cluster is deprecated and removed from all-clusters
"Test_TC_SMOKECO_2_6.yaml", # chip-repl does not support local timeout (07/20/2023) and test assumes
# TestEventTriggersEnabled is true, which it's not in CI. Also, test
# has wrong key for eventNumber: because using the right key leads to
# codegen that does not compile.
"Test_TC_SMOKECO_2_2.yaml", # chip-repl does not support local timeout (07/20/2023) and test assumes
# TestEventTriggersEnabled is true, which it's not in CI. Also, test
# has wrong key for eventNumber: because using the right key leads to
# codegen that does not compile.
"Test_TC_SMOKECO_2_3.yaml", # chip-repl does not support local timeout (07/20/2023) and test assumes
# TestEventTriggersEnabled is true, which it's not in CI. Also, test
# has wrong key for eventNumber: because using the right key leads to
# codegen that does not compile.
"Test_TC_SMOKECO_2_4.yaml", # chip-repl does not support local timeout (07/20/2023) and test assumes
# TestEventTriggersEnabled is true, which it's not in CI. Also, test
# has wrong key for eventNumber: because using the right key leads to
# codegen that does not compile.
"Test_TC_SMOKECO_2_5.yaml", # chip-repl does not support local timeout (07/20/2023) and test uses unknown
# keepSubscriptions key in the YAML. Also, test has wrong key for eventNumber:
# because using the right key leads to codegen that does not compile.
}


def _GetChipReplUnsupportedTests() -> Set[str]:
"""Tests that fail in chip-repl for some reason"""
return {
"Test_AddNewFabricFromExistingFabric.yaml", # chip-repl does not support GetCommissionerRootCertificate and IssueNocChain command
"TestEqualities.yaml", # chip-repl does not support pseudo-cluster commands that return a value
"TestExampleCluster.yaml", # chip-repl does not load custom pseudo clusters
"Test_TC_TIMESYNC_1_1.yaml", # Time sync SDK is not yet ready
"Test_TC_TIMESYNC_2_3.yaml", # Time sync SDK is not yet ready
"TestAttributesById.yaml", # chip-repl does not support AnyCommands (06/06/2023)
"TestCommandsById.yaml", # chip-repl does not support AnyCommands (06/06/2023)
"TestEventsById.yaml", # chip-repl does not support AnyCommands (06/06/2023)
"TestReadNoneSubscribeNone.yaml", # chip-repl does not support AnyCommands (07/27/2023)
"Test_TC_DRLK_2_8.yaml", # Test fails only in chip-repl: Refer--> https://github.com/project-chip/connectedhomeip/pull/27011#issuecomment-1593339855
"Test_TC_ACE_1_6.yaml", # Test fails only in chip-repl: Refer--> https://github.com/project-chip/connectedhomeip/pull/27910#issuecomment-1632485584
"Test_TC_PSCFG_1_1.yaml", # Power source configuration cluster is deprecated and removed from all-clusters
"Test_TC_PSCFG_2_1.yaml", # Power source configuration cluster is deprecated and removed from all-clusters
"Test_TC_PSCFG_2_2.yaml", # Power source configuration cluster is deprecated and removed from all-clusters
"Test_TC_SMOKECO_2_2.yaml", # chip-repl does not support local timeout (07/20/2023)
"Test_TC_SMOKECO_2_3.yaml", # chip-repl does not support local timeout (07/20/2023)
"Test_TC_SMOKECO_2_4.yaml", # chip-repl does not support local timeout (07/20/2023)
"Test_TC_SMOKECO_2_5.yaml", # chip-repl does not support local timeout (07/20/2023)
"Test_TC_SMOKECO_2_6.yaml", # chip-repl does not support local timeout (07/20/2023)
"Test_TC_IDM_1_2.yaml", # chip-repl does not support AnyCommands (19/07/2023)
"TestGroupKeyManagementCluster.yaml", # chip-repl does not support EqualityCommands (2023-08-04)
}


Expand Down Expand Up @@ -185,19 +206,23 @@ def target_for_name(name: str):
return TestTarget.ALL_CLUSTERS


def tests_with_command(chip_tool: str, is_manual: bool):
def tests_with_command(chip_tool: str, is_manual: bool, is_chip_tool_python_only: bool = False):
"""Executes `chip_tool` binary to see what tests are available, using cmd
to get the list.
"""
cmd = "list"
if is_manual:
cmd += "-manual"
elif is_chip_tool_python_only:
cmd += "-python-runner-only"

result = subprocess.run([chip_tool, "tests", cmd], capture_output=True)

test_tags = set()
if is_manual:
test_tags.add(TestTag.MANUAL)
if is_chip_tool_python_only:
test_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)

in_development_tests = [s.replace(".yaml", "") for s in _GetInDevelopmentTests()]

Expand All @@ -222,6 +247,7 @@ def _hardcoded_python_yaml_tests():
flaky_tests = _GetFlakyTests()
slow_tests = _GetSlowTests()
in_development_tests = _GetInDevelopmentTests()
chip_repl_unsupported_tests = _GetChipReplUnsupportedTests()

for path in _AllYamlTests():
if not _IsValidYamlTest(path.name):
Expand All @@ -240,6 +266,9 @@ def _hardcoded_python_yaml_tests():
if path.name in in_development_tests:
tags.add(TestTag.IN_DEVELOPMENT)

if path.name in chip_repl_unsupported_tests:
tags.add(TestTag.IN_DEVELOPMENT)

yield TestDefinition(
run_name=str(path),
name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
Expand All @@ -260,6 +289,9 @@ def AllChipToolTests(chip_tool: str):
for test in tests_with_command(chip_tool, is_manual=True):
yield test

for test in tests_with_command(chip_tool, is_manual=False, is_chip_tool_python_only=True):
yield test


__all__ = [
"TestTarget",
Expand Down
1 change: 1 addition & 0 deletions scripts/tests/chiptest/test_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ class TestTag(Enum):
SLOW = auto() # test uses Sleep and is generally slow (>=10s is a typical threshold)
FLAKY = auto() # test is considered flaky (usually a bug/time dependent issue)
IN_DEVELOPMENT = auto() # test may not pass or undergoes changes
CHIP_TOOL_PYTHON_ONLY = auto() # test uses YAML features only supported by the CHIP_TOOL_PYTHON runner.

def to_s(self):
for (k, v) in TestTag.__members__.items():
Expand Down
3 changes: 3 additions & 0 deletions scripts/tests/run_test_suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,9 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
TestTag.FLAKY,
}

if runtime != TestRunTime.CHIP_TOOL_PYTHON:
exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)

if 'all' not in target:
tests = []
for name in target:
Expand Down
2 changes: 1 addition & 1 deletion scripts/tools/zap/zap_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
# Use scripts/tools/zap/version_update.py to manage ZAP versioning as many
# files may need updating for versions
#
MIN_ZAP_VERSION = '2023.8.1'
MIN_ZAP_VERSION = '2023.8.4'


class ZapTool:
Expand Down
8 changes: 6 additions & 2 deletions src/app/tests/suites/TestEqualities.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@ config:
nodeId: 0x12344321
cluster: "EqualityCommands"
endpoint: 1
expectedValue: 20003
unexpectedValue: 20004
expectedValue:
type: INT16U
defaultValue: 20003
unexpectedValue:
type: INT16U
defaultValue: 20004

tests:
- label: "Wait for the commissioned device to be retrieved"
Expand Down
63 changes: 31 additions & 32 deletions src/app/tests/suites/TestGroupKeyManagementCluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,30 +74,29 @@ tests:
constraints:
minValue: 3

# TODO(#28396): Re-enabled when CI wouldn't break on this syntax
#
# command: "readAttribute"
# attribute: "ClusterRevision"
# response:
# saveAs: ClusterRevisionValue
# constraints:
# minValue: 1
# maxValue: 2
# type: int16u

# - label: "Creates a new variable to hold if the cluster revision is == 2"
# cluster: EqualityCommands
# command: UnsignedNumberEquals
# arguments:
# values:
# - name: Value1
# value: ClusterRevisionValue
# - name: Value2
# value: 2
# response:
# - values:
# - name: Equals
# saveAs: ClusterIsRevision2
- label: "Read the ClusterRevision attribute"
command: "readAttribute"
attribute: "ClusterRevision"
response:
saveAs: ClusterRevisionValue
constraints:
minValue: 1
maxValue: 2
type: int16u

- label: "Creates a new variable to hold if the cluster revision is == 2"
cluster: EqualityCommands
command: UnsignedNumberEquals
arguments:
values:
- name: Value1
value: ClusterRevisionValue
- name: Value2
value: 2
response:
- values:
- name: Equals
saveAs: ClusterIsRevision2

########### KeySetWrite Epoch0 field validations
- label: "KeySetWrite with EpochKey0 null fails INVALID_COMMAND"
Expand Down Expand Up @@ -161,7 +160,7 @@ tests:
- label:
"KeySetWrite with EpochKey0 with length 1 != 16 fails with
CONSTRAINT_ERROR"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
command: "KeySetWrite"
arguments:
values:
Expand All @@ -183,7 +182,7 @@ tests:
- label:
"KeySetWrite with EpochKey0 with length 0 != 16 fails with
CONSTRAINT_ERROR"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
command: "KeySetWrite"
arguments:
values:
Expand All @@ -204,7 +203,7 @@ tests:

########### KeySetWrite Epoch1 field validations
- label: "KeySetWrite with EpochStartTime1 null fails INVALID_COMMAND"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
command: "KeySetWrite"
arguments:
values:
Expand Down Expand Up @@ -245,7 +244,7 @@ tests:
- label:
"KeySetWrite with EpochKey1 with length 1 != 16 fails with
CONSTRAINT_ERROR"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
command: "KeySetWrite"
arguments:
values:
Expand All @@ -267,7 +266,7 @@ tests:
- label:
"KeySetWrite with EpochKey1 with length 0 != 16 fails with
CONSTRAINT_ERROR"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
command: "KeySetWrite"
arguments:
values:
Expand Down Expand Up @@ -310,7 +309,7 @@ tests:
########### KeySetWrite Epoch2 field validations
- label: "KeySetWrite with EpochStartTime2 null fails INVALID_COMMAND"
command: "KeySetWrite"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
arguments:
values:
- name: "GroupKeySet"
Expand Down Expand Up @@ -350,7 +349,7 @@ tests:
- label:
"KeySetWrite with EpochKey2 with length 1 != 16 fails with
CONSTRAINT_ERROR"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
command: "KeySetWrite"
arguments:
values:
Expand All @@ -372,7 +371,7 @@ tests:
- label:
"KeySetWrite with EpochKey2 with length 0 != 16 fails with
CONSTRAINT_ERROR"
# runIf: ClusterIsRevision2
runIf: ClusterIsRevision2
command: "KeySetWrite"
arguments:
values:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ tests:
minInterval: 3
maxInterval: 30
response:
Value: 0
value: 0
constraints:
type: enum8

Expand Down
1 change: 0 additions & 1 deletion src/app/tests/suites/ciTests.json
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,6 @@
"Groups": [
"TestGroupMessaging",
"TestGroupsCluster",
"TestGroupKeyManagementCluster",
"Test_TC_G_1_1",
"Test_TC_G_2_1"
],
Expand Down
5 changes: 5 additions & 0 deletions src/app/tests/suites/pythonRunnerOnlyTests.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"Groups": ["TestGroupKeyManagementCluster"],
"Others": ["TestEqualities"],
"collection": ["Groups", "Others"]
}
Loading