From e139dee954c8e8b9dabbe47b65aed82e6bd65653 Mon Sep 17 00:00:00 2001 From: C Freeman Date: Tue, 7 May 2024 08:23:51 -0400 Subject: [PATCH] YAML checker for CI - ensure no unit testing cluster (#33261) * YAML: add checker to run in CI * DNS: make a small change to a yaml test to see the CI run * Use python3, fix name * I have no idea what I'm doing * seriously, cecille... * maybe? * trial and error is just like knowing what you're doing * Restyled by prettier-yaml * Restyled by isort * Revert "Restyled by prettier-yaml" This reverts commit fc1825bda33f2cb7cf7b475f1a7f93282336ee6a. * Revert "DNS: make a small change to a yaml test to see the CI run" This reverts commit e3b66ec358922e7acbc352dc1cdf916a5c737b36. * lint * Wrong default --------- Co-authored-by: Restyled.io --- .github/workflows/cert_test_checks.yaml | 35 ++++++++++++++++ scripts/tests/chiptest/__init__.py | 4 +- scripts/tests/matter_yaml_linter.py | 56 +++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/cert_test_checks.yaml create mode 100644 scripts/tests/matter_yaml_linter.py diff --git a/.github/workflows/cert_test_checks.yaml b/.github/workflows/cert_test_checks.yaml new file mode 100644 index 00000000000000..44d545a6636897 --- /dev/null +++ b/.github/workflows/cert_test_checks.yaml @@ -0,0 +1,35 @@ +# Copyright (c) 2024 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Certification test checks + +on: + pull_request: + paths: + - "src/app/tests/suites/certification/**" + +jobs: + check-certification-tests: + name: Check for common problems in certification tests + runs-on: ubuntu-latest + + container: + image: ghcr.io/project-chip/chip-build + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Run checks + run: | + python3 scripts/tests/matter_yaml_linter.py diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 623017e2e2dd97..9ae18675cadf78 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -420,8 +420,8 @@ def AllReplYamlTests(): yield test -def AllChipToolYamlTests(): - for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, treat_chip_tool_unsupported_as_in_development=True, use_short_run_name=True): +def AllChipToolYamlTests(use_short_run_name: bool = True): + for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, treat_chip_tool_unsupported_as_in_development=True, use_short_run_name=use_short_run_name): yield test diff --git a/scripts/tests/matter_yaml_linter.py b/scripts/tests/matter_yaml_linter.py new file mode 100644 index 00000000000000..4fdcb765ce6e0c --- /dev/null +++ b/scripts/tests/matter_yaml_linter.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2024 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re +import sys +from pathlib import Path + +from chiptest import AllChipToolYamlTests + +DEFAULT_CHIP_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) + +# TODO: These tests need to be re-written. Please see https://github.com/project-chip/connectedhomeip/issues/32620 +KNOWN_BAD_UNIT_TESTING = set(('Test_TC_S_2_2.yaml', 'Test_TC_S_2_3.yaml')) + + +def _is_cert_test(path): + return "certification" in os.path.dirname(path) + + +def main(): + bad_tests = set() + for test in AllChipToolYamlTests(use_short_run_name=False): + with open(test.run_name, "r") as f: + # Unit testing cluster is disallowed in cert tests, but permissible in general integration tests + unit_test_lines = {} + if _is_cert_test(test.run_name): + unit_test_lines = {lineno: line.strip() for lineno, line in enumerate( + f) if re.search('cluster: "Unit Testing"', line)} + if unit_test_lines: + print( + f'Found certification test using Unit Testing cluster: {test.name}') + for line, val in unit_test_lines.items(): + print(f'\t{line+1}: {val}') + bad_tests.add(Path(test.run_name).name) + + if bad_tests - KNOWN_BAD_UNIT_TESTING: + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main())