From 1342608a8eb63e46f839efd17cc09254d4106f14 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Fri, 24 Feb 2023 19:23:09 +0100 Subject: [PATCH] Add python YAML runner script to runs test over chip-tool and the placeholder apps (#25155) * [chip-tool] Add chip-tool adapter for running tests with matter_yamltests * [examples/placeholder] Add placeholder adapter for running tests with matter_yamltests * Add python YAML runner script to runs test over chip-tool and the placeholder apps * Add chip-tool over python runs to the test suite * Use a scoped lock to make Tsan happy * [chip-tool] Update DiscoverCommissionablesCommand to avoid a data race when cleaning up parameters * [chip-tool] Do not call ErrorStr after RunCommand in interactive mode as ErrorStr use a single static buffer to compute the error string and in interactive mode the stack continue to do some work and it may race --- .github/workflows/tests.yaml | 47 +- .../chip-tool/commands/common/CHIPCommand.h | 8 + .../chip-tool/commands/common/Commands.cpp | 4 +- .../DiscoverCommissionablesCommand.cpp | 33 +- .../discover/DiscoverCommissionablesCommand.h | 3 + .../interactive/InteractiveCommands.cpp | 62 ++- .../py_matter_chip_tool_adapter/BUILD.gn | 38 ++ .../matter_chip_tool_adapter/__init__.py | 0 .../matter_chip_tool_adapter/adapter.py | 28 ++ .../matter_chip_tool_adapter/decoder.py | 313 +++++++++++++ .../matter_chip_tool_adapter/encoder.py | 323 ++++++++++++++ .../pyproject.toml | 17 + .../py_matter_chip_tool_adapter/setup.cfg | 23 + .../py_matter_chip_tool_adapter/setup.py | 20 + .../websocket-server/WebSocketServer.cpp | 82 +++- .../common/websocket-server/WebSocketServer.h | 10 +- .../py_matter_placeholder_adapter/BUILD.gn | 36 ++ .../matter_placeholder_adapter/__init__.py | 0 .../matter_placeholder_adapter/adapter.py | 101 +++++ .../pyproject.toml | 17 + .../py_matter_placeholder_adapter/setup.cfg | 23 + .../py_matter_placeholder_adapter/setup.py | 20 + scripts/py_matter_yamltests/BUILD.gn | 7 + .../matter_yamltests/adapter.py | 43 ++ .../matter_yamltests/hooks.py | 252 +++++++++++ .../matter_yamltests/parser_builder.py | 104 +++++ .../matter_yamltests/parser_config.py | 35 ++ .../matter_yamltests/runner.py | 207 +++++++++ .../matter_yamltests/websocket_runner.py | 105 +++++ .../test_parser_builder.py | 207 +++++++++ scripts/setup/requirements.txt | 1 + scripts/tests/chiptest/linux.py | 1 + scripts/tests/chiptest/test_definition.py | 15 +- scripts/tests/run_test_suite.py | 33 +- scripts/tests/yaml/__init__.py | 0 scripts/tests/yaml/chiptool.py | 102 +++++ scripts/tests/yaml/paths_finder.py | 101 +++++ scripts/tests/yaml/relative_importer.py | 43 ++ scripts/tests/yaml/runner.py | 324 ++++++++++++++ scripts/tests/yaml/tests_finder.py | 121 +++++ scripts/tests/yaml/tests_logger.py | 421 ++++++++++++++++++ 41 files changed, 3260 insertions(+), 70 deletions(-) create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/BUILD.gn create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/__init__.py create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/adapter.py create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/pyproject.toml create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/setup.cfg create mode 100644 examples/chip-tool/py_matter_chip_tool_adapter/setup.py create mode 100644 examples/placeholder/py_matter_placeholder_adapter/BUILD.gn create mode 100644 examples/placeholder/py_matter_placeholder_adapter/matter_placeholder_adapter/__init__.py create mode 100644 examples/placeholder/py_matter_placeholder_adapter/matter_placeholder_adapter/adapter.py create mode 100644 examples/placeholder/py_matter_placeholder_adapter/pyproject.toml create mode 100644 examples/placeholder/py_matter_placeholder_adapter/setup.cfg create mode 100644 examples/placeholder/py_matter_placeholder_adapter/setup.py create mode 100644 scripts/py_matter_yamltests/matter_yamltests/adapter.py create mode 100644 scripts/py_matter_yamltests/matter_yamltests/hooks.py create mode 100644 scripts/py_matter_yamltests/matter_yamltests/parser_builder.py create mode 100644 scripts/py_matter_yamltests/matter_yamltests/parser_config.py create mode 100644 scripts/py_matter_yamltests/matter_yamltests/runner.py create mode 100644 scripts/py_matter_yamltests/matter_yamltests/websocket_runner.py create mode 100644 scripts/py_matter_yamltests/test_parser_builder.py create mode 100644 scripts/tests/yaml/__init__.py create mode 100755 scripts/tests/yaml/chiptool.py create mode 100755 scripts/tests/yaml/paths_finder.py create mode 100644 scripts/tests/yaml/relative_importer.py create mode 100755 scripts/tests/yaml/runner.py create mode 100755 scripts/tests/yaml/tests_finder.py create mode 100755 scripts/tests/yaml/tests_logger.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 19aaf2c7c8e44c..8fb92717e6c65b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -32,7 +32,7 @@ env: jobs: test_suites_linux: name: Test Suites - Linux - timeout-minutes: 145 + timeout-minutes: 180 strategy: matrix: @@ -226,13 +226,32 @@ jobs: --tv-app ./out/linux-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \ --bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \ " + + - name: Run Tests using the python parser sending commands to chip-tool + timeout-minutes: 65 + run: | + ./scripts/run_in_build_env.sh \ + "./scripts/tests/run_test_suite.py \ + --runner chip_tool_python \ + --chip-tool ./out/linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \ + run \ + --iterations 1 \ + --test-timeout-seconds 120 \ + --all-clusters-app ./out/linux-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \ + --lock-app ./out/linux-x64-lock-${BUILD_VARIANT}/chip-lock-app \ + --ota-provider-app ./out/linux-x64-ota-provider-${BUILD_VARIANT}/chip-ota-provider-app \ + --ota-requestor-app ./out/linux-x64-ota-requestor-${BUILD_VARIANT}/chip-ota-requestor-app \ + --tv-app ./out/linux-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \ + --bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \ + " + - name: Run Tests using chip-repl (skip slow) timeout-minutes: 45 if: github.event_name == 'pull_request' run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ - --run-yamltests-with-chip-repl \ + --runner chip_repl_python \ --exclude-tags MANUAL \ --exclude-tags FLAKY \ --exclude-tags IN_DEVELOPMENT \ @@ -253,7 +272,7 @@ jobs: run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ - --run-yamltests-with-chip-repl \ + --runner chip_repl_python \ run \ --iterations 1 \ --test-timeout-seconds 120 \ @@ -283,7 +302,7 @@ jobs: test_suites_darwin: name: Test Suites - Darwin - timeout-minutes: 150 + timeout-minutes: 180 strategy: matrix: @@ -372,6 +391,26 @@ jobs: --tv-app ./out/darwin-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \ --bridge-app ./out/darwin-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \ " + + - name: Run Tests using the python parser sending commands to chip-tool + timeout-minutes: 80 + run: | + ./scripts/run_in_build_env.sh \ + "./scripts/tests/run_test_suite.py \ + --runner chip_tool_python \ + --chip-tool ./out/darwin-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT}/chip-tool \ + --target-skip-glob '{Test_TC_DGTHREAD_2_1,Test_TC_DGTHREAD_2_2,Test_TC_DGTHREAD_2_3,Test_TC_DGTHREAD_2_4}' \ + run \ + --iterations 1 \ + --test-timeout-seconds 120 \ + --all-clusters-app ./out/darwin-x64-all-clusters-${BUILD_VARIANT}/chip-all-clusters-app \ + --lock-app ./out/darwin-x64-lock-${BUILD_VARIANT}/chip-lock-app \ + --ota-provider-app ./out/darwin-x64-ota-provider-${BUILD_VARIANT}/chip-ota-provider-app \ + --ota-requestor-app ./out/darwin-x64-ota-requestor-${BUILD_VARIANT}/chip-ota-requestor-app \ + --tv-app ./out/darwin-x64-tv-app-${BUILD_VARIANT}/chip-tv-app \ + --bridge-app ./out/darwin-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \ + " + - name: Uploading core files uses: actions/upload-artifact@v3 if: ${{ failure() && !env.ACT }} diff --git a/examples/chip-tool/commands/common/CHIPCommand.h b/examples/chip-tool/commands/common/CHIPCommand.h index cbe09472f3ea8d..265275416474f3 100644 --- a/examples/chip-tool/commands/common/CHIPCommand.h +++ b/examples/chip-tool/commands/common/CHIPCommand.h @@ -94,6 +94,14 @@ class CHIPCommand : public Command void SetCommandExitStatus(CHIP_ERROR status) { mCommandExitStatus = status; + // In interactive mode the stack is not shut down once a command is ended. + // That means calling `ErrorStr(err)` from the main thread when command + // completion is signaled may race since `ErrorStr` uses a static sErrorStr + // buffer for computing the error string. Call it here instead. + if (IsInteractive() && CHIP_NO_ERROR != status) + { + ChipLogError(chipTool, "Run command failure: %s", chip::ErrorStr(status)); + } StopWaiting(); } diff --git a/examples/chip-tool/commands/common/Commands.cpp b/examples/chip-tool/commands/common/Commands.cpp index ef4386725ec2f2..1276e4a624220b 100644 --- a/examples/chip-tool/commands/common/Commands.cpp +++ b/examples/chip-tool/commands/common/Commands.cpp @@ -181,9 +181,7 @@ int Commands::RunInteractive(const char * command) delete[] argv[i]; } - VerifyOrReturnValue(CHIP_NO_ERROR == err, EXIT_FAILURE, ChipLogError(chipTool, "Run command failure: %s", chip::ErrorStr(err))); - - return EXIT_SUCCESS; + return (err == CHIP_NO_ERROR) ? EXIT_SUCCESS : EXIT_FAILURE; } CHIP_ERROR Commands::RunCommand(int argc, char ** argv, bool interactive) diff --git a/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.cpp b/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.cpp index ad474039d07f65..b8b7ca6feba757 100644 --- a/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.cpp +++ b/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.cpp @@ -29,49 +29,56 @@ void DiscoverCommissionablesCommandBase::OnDiscoveredDevice(const chip::Dnssd::D if (mDiscoverOnce.ValueOr(true)) { - CurrentCommissioner().StopCommissionableDiscovery(); + mCommissioner->RegisterDeviceDiscoveryDelegate(nullptr); + mCommissioner->StopCommissionableDiscovery(); SetCommandExitStatus(CHIP_NO_ERROR); } } CHIP_ERROR DiscoverCommissionablesCommand::RunCommand() { - CurrentCommissioner().RegisterDeviceDiscoveryDelegate(this); + mCommissioner = &CurrentCommissioner(); + mCommissioner->RegisterDeviceDiscoveryDelegate(this); Dnssd::DiscoveryFilter filter(Dnssd::DiscoveryFilterType::kNone, (uint64_t) 0); - return CurrentCommissioner().DiscoverCommissionableNodes(filter); + return mCommissioner->DiscoverCommissionableNodes(filter); } CHIP_ERROR DiscoverCommissionableByShortDiscriminatorCommand::RunCommand() { - CurrentCommissioner().RegisterDeviceDiscoveryDelegate(this); + mCommissioner = &CurrentCommissioner(); + mCommissioner->RegisterDeviceDiscoveryDelegate(this); chip::Dnssd::DiscoveryFilter filter(chip::Dnssd::DiscoveryFilterType::kShortDiscriminator, mDiscriminator); - return CurrentCommissioner().DiscoverCommissionableNodes(filter); + return mCommissioner->DiscoverCommissionableNodes(filter); } CHIP_ERROR DiscoverCommissionableByLongDiscriminatorCommand::RunCommand() { - CurrentCommissioner().RegisterDeviceDiscoveryDelegate(this); + mCommissioner = &CurrentCommissioner(); + mCommissioner->RegisterDeviceDiscoveryDelegate(this); chip::Dnssd::DiscoveryFilter filter(chip::Dnssd::DiscoveryFilterType::kLongDiscriminator, mDiscriminator); - return CurrentCommissioner().DiscoverCommissionableNodes(filter); + return mCommissioner->DiscoverCommissionableNodes(filter); } CHIP_ERROR DiscoverCommissionableByCommissioningModeCommand::RunCommand() { - CurrentCommissioner().RegisterDeviceDiscoveryDelegate(this); + mCommissioner = &CurrentCommissioner(); + mCommissioner->RegisterDeviceDiscoveryDelegate(this); chip::Dnssd::DiscoveryFilter filter(chip::Dnssd::DiscoveryFilterType::kCommissioningMode); - return CurrentCommissioner().DiscoverCommissionableNodes(filter); + return mCommissioner->DiscoverCommissionableNodes(filter); } CHIP_ERROR DiscoverCommissionableByVendorIdCommand::RunCommand() { - CurrentCommissioner().RegisterDeviceDiscoveryDelegate(this); + mCommissioner = &CurrentCommissioner(); + mCommissioner->RegisterDeviceDiscoveryDelegate(this); chip::Dnssd::DiscoveryFilter filter(chip::Dnssd::DiscoveryFilterType::kVendorId, mVendorId); - return CurrentCommissioner().DiscoverCommissionableNodes(filter); + return mCommissioner->DiscoverCommissionableNodes(filter); } CHIP_ERROR DiscoverCommissionableByDeviceTypeCommand::RunCommand() { - CurrentCommissioner().RegisterDeviceDiscoveryDelegate(this); + mCommissioner = &CurrentCommissioner(); + mCommissioner->RegisterDeviceDiscoveryDelegate(this); chip::Dnssd::DiscoveryFilter filter(chip::Dnssd::DiscoveryFilterType::kDeviceType, mDeviceType); - return CurrentCommissioner().DiscoverCommissionableNodes(filter); + return mCommissioner->DiscoverCommissionableNodes(filter); } diff --git a/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.h b/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.h index b0bff0e4e113e0..74d946cea4cd76 100644 --- a/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.h +++ b/examples/chip-tool/commands/discover/DiscoverCommissionablesCommand.h @@ -36,6 +36,9 @@ class DiscoverCommissionablesCommandBase : public CHIPCommand, public chip::Cont /////////// CHIPCommand Interface ///////// chip::System::Clock::Timeout GetWaitDuration() const override { return chip::System::Clock::Seconds16(30); } +protected: + chip::Controller::DeviceCommissioner * mCommissioner; + private: chip::Optional mDiscoverOnce; }; diff --git a/examples/chip-tool/commands/interactive/InteractiveCommands.cpp b/examples/chip-tool/commands/interactive/InteractiveCommands.cpp index 854b0877df0cd5..599de21af3e8b5 100644 --- a/examples/chip-tool/commands/interactive/InteractiveCommands.cpp +++ b/examples/chip-tool/commands/interactive/InteractiveCommands.cpp @@ -43,6 +43,17 @@ void ENFORCE_FORMAT(3, 0) LoggingCallback(const char * module, uint8_t category, ClearLine(); } +class ScopedLock +{ +public: + ScopedLock(std::mutex & mutex) : mMutex(mutex) { mMutex.lock(); } + + ~ScopedLock() { mMutex.unlock(); } + +private: + std::mutex & mMutex; +}; + struct InteractiveServerResultLog { std::string module; @@ -58,14 +69,39 @@ struct InteractiveServerResult std::vector mResults; std::vector mLogs; + // The InteractiveServerResult instance (gInteractiveServerResult) is initially + // accessed on the main thread in InteractiveServerCommand::RunCommand, which is + // when chip-tool starts in 'interactive server' mode. + // + // Then command results are normally sent over the wire onto the main thread too + // when a command is received over WebSocket in InteractiveServerCommand::OnWebSocketMessageReceived + // which for most cases runs a command onto the chip thread and block until + // it is resolved (or until it timeouts). + // + // But in the meantime, when some parts of the command result happens, it is appended + // to the mResults vector onto the chip thread. + // + // For empty commands, which means that the test suite is *waiting* for some events + // (e.g a subscription report), the command results are sent over the chip thread + // (this is the isAsyncReport use case). + // + // Finally, logs can be appended from either the chip thread or the main thread. + // + // This class should be refactored to abstract that properly and reduce the scope of + // of the mutex, but in the meantime, the access to the members of this class are + // protected by a mutex. + std::mutex mMutex; + void Setup(bool isAsyncReport) { + auto lock = ScopedLock(mMutex); mEnabled = true; mIsAsyncReport = isAsyncReport; } void Reset() { + auto lock = ScopedLock(mMutex); mEnabled = false; mIsAsyncReport = false; mStatus = EXIT_SUCCESS; @@ -73,10 +109,15 @@ struct InteractiveServerResult mLogs.clear(); } - bool IsAsyncReport() const { return mIsAsyncReport; } + bool IsAsyncReport() + { + auto lock = ScopedLock(mMutex); + return mIsAsyncReport; + } void MaybeAddLog(const char * module, uint8_t category, const char * base64Message) { + auto lock = ScopedLock(mMutex); VerifyOrReturn(mEnabled); const char * messageType = nullptr; @@ -98,12 +139,16 @@ struct InteractiveServerResult void MaybeAddResult(const char * result) { + auto lock = ScopedLock(mMutex); VerifyOrReturn(mEnabled); + mResults.push_back(result); } - std::string AsJsonString() const + std::string AsJsonString() { + auto lock = ScopedLock(mMutex); + std::string resultsStr; if (mResults.size()) { @@ -205,13 +250,6 @@ CHIP_ERROR InteractiveServerCommand::RunCommand() return CHIP_NO_ERROR; } -void SendOverWebSocket(intptr_t context) -{ - auto server = reinterpret_cast(context); - server->Send(gInteractiveServerResult.AsJsonString().c_str()); - gInteractiveServerResult.Reset(); -} - bool InteractiveServerCommand::OnWebSocketMessageReceived(char * msg) { bool isAsyncReport = strlen(msg) == 0; @@ -219,7 +257,8 @@ bool InteractiveServerCommand::OnWebSocketMessageReceived(char * msg) VerifyOrReturnValue(!isAsyncReport, true); auto shouldStop = ParseCommand(msg, &gInteractiveServerResult.mStatus); - chip::DeviceLayer::PlatformMgr().ScheduleWork(SendOverWebSocket, reinterpret_cast(&mWebSocketServer)); + mWebSocketServer.Send(gInteractiveServerResult.AsJsonString().c_str()); + gInteractiveServerResult.Reset(); return shouldStop; } @@ -228,7 +267,8 @@ CHIP_ERROR InteractiveServerCommand::LogJSON(const char * json) gInteractiveServerResult.MaybeAddResult(json); if (gInteractiveServerResult.IsAsyncReport()) { - chip::DeviceLayer::PlatformMgr().ScheduleWork(SendOverWebSocket, reinterpret_cast(&mWebSocketServer)); + mWebSocketServer.Send(gInteractiveServerResult.AsJsonString().c_str()); + gInteractiveServerResult.Reset(); } return CHIP_NO_ERROR; } diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/BUILD.gn b/examples/chip-tool/py_matter_chip_tool_adapter/BUILD.gn new file mode 100644 index 00000000000000..cd0b9cc11132e7 --- /dev/null +++ b/examples/chip-tool/py_matter_chip_tool_adapter/BUILD.gn @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python.gni") + +pw_python_package("matter_chip_tool_adapter") { + setup = [ + "setup.py", + "setup.cfg", + "pyproject.toml", + ] + + sources = [ + "matter_chip_tool_adapter/__init__.py", + "matter_chip_tool_adapter/adapter.py", + "matter_chip_tool_adapter/decoder.py", + "matter_chip_tool_adapter/encoder.py", + ] + + # TODO: at a future time consider enabling all (* or missing) here to get + # pylint checking these files + static_analysis = [] +} diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/__init__.py b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/adapter.py b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/adapter.py new file mode 100644 index 00000000000000..0794b4af7ff1dd --- /dev/null +++ b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/adapter.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .decoder import Decoder +from .encoder import Encoder + + +class Adapter: + def __init__(self, specifications): + self.encoder = Encoder(specifications) + self.decoder = Decoder(specifications) + + def encode(self, request): + return self.encoder.encode(request) + + def decode(self, response): + return self.decoder.decode(response) diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py new file mode 100644 index 00000000000000..df906ec318b83e --- /dev/null +++ b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py @@ -0,0 +1,313 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json + +# These constants represent the vocabulary used for the incoming JSON. +_CLUSTER_ID = 'clusterId' +_ENDPOINT_ID = 'endpointId' +_RESPONSE_ID = 'commandId' +_ATTRIBUTE_ID = 'attributeId' +_EVENT_ID = 'eventId' + +# These constants represent the vocabulary used for the outgoing data. +_CLUSTER = 'cluster' +_ENDPOINT = 'endpoint' +_RESPONSE = 'command' +_ATTRIBUTE = 'attribute' +_EVENT = 'event' + + +# These constants represent the common vocabulary between input and output. +_ERROR = 'error' +_CLUSTER_ERROR = 'clusterError' +_VALUE = 'value' + +# FabricIndex is a special case where the field is added as a struct field by the SDK +# if needed but is not part of the XML definition of the struct. +# These constants are used to map from the field code (254) to the field name if such +# a field is used for a fabric scoped struct. +_FABRIC_INDEX_FIELD_CODE = '254' +_FABRIC_INDEX_FIELD_NAME = 'FabricIndex' +_FABRIC_INDEX_FIELD_TYPE = 'int8u' + + +class Decoder: + """ + This class implement decoding a test step response from the adapter format to the + matter_yamltests format. + """ + + def __init__(self, specifications): + self.__specs = specifications + self.__converter = Converter(specifications) + + def decode(self, payload): + payload, logs = self.__get_payload_content(payload) + payload = self.__translate_names(payload) + payload = self.__converter.convert(payload) + + if len(payload) == 0: + payload = [{}] + elif len(payload) > 1 and payload[-1] == {'error': 'FAILURE'}: + payload = payload[:-1] + return payload, logs + + def __translate_names(self, payloads): + translated_payloads = [] + specs = self.__specs + + for payload in payloads: + translated_payload = {} + for key, value in payload.items(): + if key == _CLUSTER_ID: + key = _CLUSTER + value = specs.get_cluster_name(value) + elif key == _ENDPOINT_ID: + key = _ENDPOINT + elif key == _RESPONSE_ID: + key = _RESPONSE + value = specs.get_response_name( + payload[_CLUSTER_ID], value) + elif key == _ATTRIBUTE_ID: + key = _ATTRIBUTE + value = specs.get_attribute_name( + payload[_CLUSTER_ID], value) + elif key == _EVENT_ID: + key = _EVENT + value = specs.get_event_name(payload[_CLUSTER_ID], value) + elif key == _VALUE or key == _ERROR or key == _CLUSTER_ERROR: + pass + else: + # Raise an error since the other fields probably needs to be translated too. + raise KeyError(f'Error: field "{key}" not supported') + + if value is None and (key == _CLUSTER or key == _RESPONSE or key == _ATTRIBUTE or key == _EVENT): + # If the definition for this cluster/command/attribute/event is missing, there is not + # much we can do to convert the response to the proper format. It usually indicates that + # the cluster definition is missing something. So we just raise an exception to tell the + # user something is wrong and the cluster definition needs to be updated. + cluster_code = hex(payload[_CLUSTER_ID]) + if key == _CLUSTER: + raise KeyError( + f'Error: The cluster ({cluster_code}) definition can not be found. Please update the cluster definition.') + else: + value_code = hex(payload[key + 'Id']) + raise KeyError( + f'Error: The cluster ({cluster_code}) {key} ({value_code}) definition can not be found. Please update the cluster definition.') + + translated_payload[key] = value + translated_payloads.append(translated_payload) + + return translated_payloads + + def __get_payload_content(self, payload): + json_payload = json.loads(payload) + results = json_payload.get('results') + logs = MatterLog.decode_logs(json_payload.get('logs')) + return results, logs + + +class MatterLog: + def __init__(self, log): + self.module = log['module'] + self.level = log['category'] + + base64_message = log["message"].encode('utf-8') + decoded_message_bytes = base64.b64decode(base64_message) + # TODO We do assume utf-8 encoding is used, it may not be true though. + self.message = decoded_message_bytes.decode('utf-8') + + def decode_logs(logs): + return list(map(MatterLog, logs)) + + +class Converter(): + """ + This class converts between the JSON representation used by chip-tool to transmit + information and the response format expected by the test suite. + + There is not much differences and ideally we won't have to do any conversion. + For example chip-tool could do the field name mapping directly instead of relying on + the adapter, or floats can be converted to the right format directly. But in the + meantime the conversion is done here. + """ + + def __init__(self, specifications): + self.__specs = specifications + self.__converters = [ + StructFieldsNameConverter(), + FloatConverter(), + OctetStringConverter() + ] + + def convert(self, payloads): + return [self._convert(payload) for payload in payloads] + + def _convert(self, rv): + if _VALUE not in rv or _CLUSTER not in rv: + return rv + + if _RESPONSE in rv: + out_value = self.__convert_command(rv) + elif _ATTRIBUTE in rv: + out_value = self.__convert_attribute(rv) + elif _EVENT in rv: + out_value = self.__convert_event(rv) + else: + out_value = rv[_VALUE] + + rv[_VALUE] = out_value + return rv + + def __convert_command(self, rv): + specs = self.__specs + cluster_name = rv[_CLUSTER] + response_name = rv[_RESPONSE] + value = rv[_VALUE] + + response = specs.get_response_by_name(cluster_name, response_name) + if not response: + raise KeyError(f'Error: response "{response_name}" not found.') + + typename = response.name + array = False + return self.__run(value, cluster_name, typename, array) + + def __convert_attribute(self, rv): + specs = self.__specs + cluster_name = rv[_CLUSTER] + attribute_name = rv[_ATTRIBUTE] + value = rv[_VALUE] + + attribute = specs.get_attribute_by_name(cluster_name, attribute_name) + if not attribute: + raise KeyError(f'Error: attribute "{attribute_name}" not found.') + + typename = attribute.definition.data_type.name + array = attribute.definition.is_list + return self.__run(value, cluster_name, typename, array) + + def __convert_event(self, rv): + specs = self.__specs + cluster_name = rv[_CLUSTER] + event_name = rv[_EVENT] + value = rv[_VALUE] + + event = specs.get_event_by_name(cluster_name, event_name) + if not event: + raise KeyError(f'Error: event "{event_name}" not found.') + + typename = event.name + array = False + return self.__run(value, cluster_name, typename, array) + + def __run(self, value, cluster_name: str, typename: str, array: bool): + for converter in self.__converters: + value = converter.run(self.__specs, value, + cluster_name, typename, array) + return value + + +class BaseConverter: + def run(self, specs, value, cluster_name: str, typename: str, array: bool): + if isinstance(value, dict) and not array: + struct = specs.get_struct_by_name( + cluster_name, typename) or specs.get_event_by_name(cluster_name, typename) + for field in struct.fields: + field_name = field.name + field_type = field.data_type.name + field_array = field.is_list + if field_name in value: + value[field_name] = self.run( + specs, value[field_name], cluster_name, field_type, field_array) + elif isinstance(value, list) and array: + value = [self.run(specs, v, cluster_name, typename, False) + for v in value] + elif value is not None: + value = self.maybe_convert(typename.lower(), value) + + return value + + def maybe_convert(self, typename: str, value): + return value + + +class FloatConverter(BaseConverter): + """ + Jsoncpp stores floats as double. + For float values that are just stored as an approximation it ends up with + a different output than expected when reading them back + """ + + def maybe_convert(self, typename, value): + if typename == 'single': + value = float('%g' % value) + return value + + +class OctetStringConverter(BaseConverter): + def maybe_convert(self, typename, value): + if typename == 'octet_string' or typename == 'long_octet_string': + if value == '': + value = bytes() + elif value.startswith('base64:'): + value = base64.b64decode(value.removeprefix('base64:')) + + return value + + +class StructFieldsNameConverter(): + """ + Converts fields identifiers to the field names specified in the cluster definition. + """ + + def run(self, specs, value, cluster_name: str, typename: str, array: bool): + if isinstance(value, dict) and not array: + struct = specs.get_struct_by_name( + cluster_name, typename) or specs.get_event_by_name(cluster_name, typename) + for field in struct.fields: + field_code = field.code + field_name = field.name + field_type = field.data_type.name + field_array = field.is_list + # chip-tool returns the field code as an integer but the test suite expects + # a field name. + # To not confuse the test suite, the field code is replaced by its field name + # equivalent and then removed. + if str(field_code) in value: + value[field_name] = self.run( + specs, + value[str(field_code)], + cluster_name, + field_type, + field_array + ) + del value[str(field_code)] + + if specs.is_fabric_scoped(struct): + value[_FABRIC_INDEX_FIELD_NAME] = self.run( + specs, + value[_FABRIC_INDEX_FIELD_CODE], + cluster_name, + _FABRIC_INDEX_FIELD_TYPE, + False) + del value[_FABRIC_INDEX_FIELD_CODE] + + elif isinstance(value, list) and array: + value = [self.run(specs, v, cluster_name, typename, False) + for v in value] + + return value diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py new file mode 100644 index 00000000000000..ffba568adb644b --- /dev/null +++ b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py @@ -0,0 +1,323 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +import re + +_ALIASES = { + 'CommissionerCommands': { + 'alias': 'pairing', + 'commands': { + 'PairWithCode': { + 'alias': 'code', + 'arguments': { + 'nodeId': 'node-id', + 'discoverOnce': 'discover-once', + }, + 'has_destination': False, + 'has_endpoint': False, + }, + 'GetCommissionerNodeId': { + 'has_destination': False, + 'has_endpoint': False, + } + } + }, + + 'DelayCommands': { + 'alias': 'delay', + 'commands': { + 'WaitForCommissionee': { + 'arguments': { + 'expireExistingSession': 'expire-existing-session', + }, + 'has_destination': False, + 'has_endpoint': False, + } + } + }, + + 'DiscoveryCommands': { + 'alias': 'discover', + 'commands': { + 'FindCommissionable': { + 'alias': 'commissionables', + 'has_destination': False, + 'has_endpoint': False, + }, + 'FindCommissionableByShortDiscriminator': { + 'has_destination': False, + 'has_endpoint': False, + }, + 'FindCommissionableByLongDiscriminator': { + 'has_destination': False, + 'has_endpoint': False, + }, + 'FindCommissionableByVendorId': { + 'has_destination': False, + 'has_endpoint': False, + }, + 'FindCommissionableByDeviceType': { + 'has_destination': False, + 'has_endpoint': False, + }, + 'FindCommissionableByCommissioningMode': { + 'has_destination': False, + 'has_endpoint': False, + }, + } + } +} + +_GLOBAL_ALIASES = { + 'readAttribute': 'read', + 'writeAttribute': 'write', + 'subscribeAttribute': 'subscribe', + 'readEvent': 'read-event', + 'subscribeEvent': 'subscribe-event', +} + + +class Encoder: + """ + This class converts the names from the YAML tests to the chip-tool equivalent. + """ + + def __init__(self, specifications): + self.__specs = specifications + + def encode(self, request): + cluster = self.__get_cluster_name(request) + command, command_specifier = self.__get_command_name(request) + + if command == 'wait-for-report': + return '' + + arguments = self.__get_arguments(request) + base64_arguments = base64.b64encode( + (f'{{ {arguments} }}').encode('utf-8')).decode('utf-8') + + payload = f'"cluster": "{cluster}", "command": "{command}", "arguments" : "base64:{base64_arguments}"' + if command_specifier: + payload += f', "command_specifier": "{command_specifier}"' + return f'json:{{ {payload} }}' + + def __get_cluster_name(self, request): + return self.__get_alias(request.cluster) or self.__format_cluster_name(request.cluster) + + def __get_command_name(self, request): + command_name = self.__get_alias( + request.cluster, request.command) or self.__format_command_name(request.command) + + # 'readAttribute' is converted to 'read attr-name', 'writeAttribute' is converted to 'write attr-name', + # 'readEvent' is converted to 'read event-name', etc. + if request.is_attribute: + command_specifier = self.__format_command_name(request.attribute) + # chip-tool exposes writable attribute under the "write" command, but for non-writable + # attributes, those appear under the "force-write" command. + if command_name == 'write': + attribute = self.__specs.get_attribute_by_name( + request.cluster, request.attribute) + if attribute and not attribute.is_writable: + command_name = 'force-write' + elif request.is_event: + command_specifier = self.__format_command_name(request.event) + else: + command_specifier = None + + return command_name, command_specifier + + def __get_arguments(self, request): + arguments = '' + arguments = self.__maybe_add_destination(arguments, request) + arguments = self.__maybe_add_endpoint(arguments, request) + arguments = self.__maybe_add_command_arguments(arguments, request) + arguments = self.__maybe_add( + arguments, request.min_interval, "min-interval") + arguments = self.__maybe_add( + arguments, request.max_interval, "max-interval") + arguments = self.__maybe_add(arguments, request.timed_interaction_timeout_ms, + "timedInteractionTimeoutMs") + arguments = self.__maybe_add( + arguments, request.event_number, "event-min") + arguments = self.__maybe_add( + arguments, request.busy_wait_ms, "busyWaitForMs") + arguments = self.__maybe_add( + arguments, request.identity, "commissioner-name") + arguments = self.__maybe_add(arguments, request.fabric_filtered, + "fabric-filtered") + + return arguments + + def __maybe_add_destination(self, rv, request): + if not self._supports_destination(request): + return rv + + destination_argument_name = 'destination-id' + destination_argument_value = None + + if request.group_id: + destination_argument_value = hex( + 0xffffffffffff0000 | int(request.group_id)) + elif request.node_id: + destination_argument_value = hex(request.node_id) + else: + destination_argument_value = None + + if rv: + rv += ', ' + rv += f'"{destination_argument_name}": "{destination_argument_value}"' + return rv + + def __maybe_add_endpoint(self, rv, request): + if not self._supports_endpoint(request): + return rv + + endpoint_argument_name = 'endpoint-id-ignored-for-group-commands' + endpoint_argument_value = request.endpoint + + if (request.is_attribute and not request.command == "writeAttribute") or request.is_event: + endpoint_argument_name = 'endpoint-ids' + + if rv: + rv += ', ' + rv += f'"{endpoint_argument_name}": "{endpoint_argument_value}"' + return rv + + def __maybe_add_command_arguments(self, rv, request): + if request.arguments is None: + return rv + + for entry in request.arguments['values']: + name = self.__get_argument_name(request, entry) + value = self.__encode_value(entry['value']) + if rv: + rv += ', ' + rv += f'"{name}":{value}' + + return rv + + def __get_argument_name(self, request, entry): + cluster_name = request.cluster + command_name = request.command + argument_name = entry.get('name') + + if request.is_attribute: + if command_name == 'writeAttribute': + argument_name = 'attribute-values' + else: + argument_name = 'value' + + return self.__get_alias(cluster_name, command_name, argument_name) or argument_name + + def __maybe_add(self, rv, value, name): + if value is None: + return rv + + if rv: + rv += ', ' + rv += f'"{name}":"{value}"' + return rv + + def __encode_value(self, value): + value = self.__encode_octet_strings(value) + value = self.__lower_camel_case_member_fields(value) + return self.__convert_to_json_string(value) + + def __encode_octet_strings(self, value): + if isinstance(value, list): + value = [self.__encode_octet_strings(entry) for entry in value] + elif isinstance(value, dict): + value = {key: self.__encode_octet_strings( + value[key]) for key in value} + elif isinstance(value, bytes): + value = 'hex:' + ''.join("{:02x}".format(c) for c in value) + return value + + def __lower_camel_case_member_fields(self, value): + if isinstance(value, list): + value = [self.__lower_camel_case_member_fields( + entry) for entry in value] + elif isinstance(value, dict): + value = {self.__to_lower_camel_case( + key): self.__lower_camel_case_member_fields(value[key]) for key in value} + return value + + def __convert_to_json_string(self, value): + is_str = isinstance(value, str) + value = json.dumps(value) + if not is_str: + value = value.replace("\"", "\\\"") + value = f'"{value}"' + + return value + + def __to_lower_camel_case(self, name): + return name[:1].lower() + name[1:] + + def __format_cluster_name(self, name): + return name.lower().replace(' ', '').replace('/', '').lower() + + def __format_command_name(self, name): + if name is None: + return name + + return re.sub(r'([a-z])([A-Z])', r'\1-\2', name).replace(' ', '-').replace(':', '-').replace('/', '').replace('_', '-').lower() + + def __get_alias(self, cluster_name: str, command_name: str = None, argument_name: str = None): + if argument_name is None and command_name in _GLOBAL_ALIASES: + return _GLOBAL_ALIASES.get(command_name) + + aliases = _ALIASES.get(cluster_name) + if aliases is None: + return None + + if command_name is None: + return aliases.get('alias') + + aliases = aliases.get('commands') + if aliases is None or aliases.get(command_name) is None: + return None + + aliases = aliases.get(command_name) + if argument_name is None: + return aliases.get('alias') + + aliases = aliases.get('arguments') + if aliases is None or aliases.get(argument_name) is None: + return None + + return aliases.get(argument_name) + + def _supports_endpoint(self, request): + return self._has_support(request, 'has_endpoint') + + def _supports_destination(self, request): + return self._has_support(request, 'has_destination') + + def _has_support(self, request, feature_name): + aliases = _ALIASES.get(request.cluster) + if aliases is None: + return True + + aliases = aliases.get('commands') + if aliases is None: + return True + + aliases = aliases.get(request.command) + if aliases is None: + return True + + return aliases.get(feature_name, True) diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/pyproject.toml b/examples/chip-tool/py_matter_chip_tool_adapter/pyproject.toml new file mode 100644 index 00000000000000..9080987af6e7be --- /dev/null +++ b/examples/chip-tool/py_matter_chip_tool_adapter/pyproject.toml @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ['setuptools', 'wheel'] +build-backend = 'setuptools.build_meta' diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/setup.cfg b/examples/chip-tool/py_matter_chip_tool_adapter/setup.cfg new file mode 100644 index 00000000000000..833fb9291d5ab1 --- /dev/null +++ b/examples/chip-tool/py_matter_chip_tool_adapter/setup.cfg @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[metadata] +name = matter_chip_tool_adapter +version = 0.0.1 +author = Project CHIP Authors +description = chip-tool adapter for matter_yamltests + +[options] +packages = find: +zip_safe = False diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/setup.py b/examples/chip-tool/py_matter_chip_tool_adapter/setup.py new file mode 100644 index 00000000000000..ed8e14d3ef40db --- /dev/null +++ b/examples/chip-tool/py_matter_chip_tool_adapter/setup.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""The matter_chip_tool_adapter package.""" + +import setuptools # type: ignore + +setuptools.setup() # Package definition in setup.cfg diff --git a/examples/common/websocket-server/WebSocketServer.cpp b/examples/common/websocket-server/WebSocketServer.cpp index 867180b976c012..49c90fab5d1ad9 100644 --- a/examples/common/websocket-server/WebSocketServer.cpp +++ b/examples/common/websocket-server/WebSocketServer.cpp @@ -21,11 +21,19 @@ #include #include +#include +#include + constexpr uint16_t kDefaultWebSocketServerPort = 9002; constexpr uint16_t kMaxMessageBufferLen = 8192; namespace { lws * gWebSocketInstance = nullptr; +std::deque gMessageQueue; + +// This mutex protect the global gMessageQueue instance such that messages +// can be added/removed from multiple threads. +std::mutex gMutex; void LogWebSocketCallbackReason(lws_callback_reasons reason) { @@ -102,29 +110,39 @@ static int OnWebSocketCallback(lws * wsi, lws_callback_reasons reason, void * us { LogWebSocketCallbackReason(reason); + WebSocketServer * server = nullptr; + auto protocol = lws_get_protocol(wsi); + if (protocol) + { + server = static_cast(protocol->user); + if (nullptr == server) + { + ChipLogError(chipTool, "Failed to retrieve the server interactive context."); + return -1; + } + } + if (LWS_CALLBACK_RECEIVE == reason) { char msg[kMaxMessageBufferLen + 1 /* for null byte */] = {}; VerifyOrDie(sizeof(msg) > len); memcpy(msg, in, len); - auto protocol = lws_get_protocol(wsi); - auto delegate = static_cast(protocol->user); - if (nullptr == delegate) - { - ChipLogError(chipTool, "Failed to retrieve the server interactive context."); - return -1; - } + server->OnWebSocketMessageReceived(msg); + } + else if (LWS_CALLBACK_SERVER_WRITEABLE == reason) + { + std::lock_guard lock(gMutex); - if (!delegate->OnWebSocketMessageReceived(msg)) + for (auto & msg : gMessageQueue) { - auto context = lws_get_context(wsi); - lws_default_loop_exit(context); + chip::Platform::ScopedMemoryBuffer buffer; + VerifyOrDie(buffer.Calloc(LWS_PRE + msg.size())); + memcpy(&buffer[LWS_PRE], (void *) msg.c_str(), msg.size()); + lws_write(wsi, &buffer[LWS_PRE], msg.size(), LWS_WRITE_TEXT); } - } - else if (LWS_CALLBACK_CLIENT_ESTABLISHED == reason) - { - lws_callback_on_writable(wsi); + + gMessageQueue.clear(); } else if (LWS_CALLBACK_ESTABLISHED == reason) { @@ -143,7 +161,7 @@ CHIP_ERROR WebSocketServer::Run(chip::Optional port, WebSocketServerDe { VerifyOrReturnError(nullptr != delegate, CHIP_ERROR_INVALID_ARGUMENT); - lws_protocols protocols[] = { { "ws", OnWebSocketCallback, 0, 0, 0, delegate, 0 }, LWS_PROTOCOL_LIST_TERM }; + lws_protocols protocols[] = { { "ws", OnWebSocketCallback, 0, 0, 0, this, 0 }, LWS_PROTOCOL_LIST_TERM }; lws_context_creation_info info; memset(&info, 0, sizeof(info)); @@ -155,18 +173,36 @@ CHIP_ERROR WebSocketServer::Run(chip::Optional port, WebSocketServerDe auto context = lws_create_context(&info); VerifyOrReturnError(nullptr != context, CHIP_ERROR_INTERNAL); - lws_context_default_loop_run_destroy(context); + mRunning = true; + mDelegate = delegate; + + while (mRunning) + { + lws_service(context, -1); + + std::lock_guard lock(gMutex); + if (gMessageQueue.size()) + { + lws_callback_on_writable(gWebSocketInstance); + } + } + lws_context_destroy(context); return CHIP_NO_ERROR; } -CHIP_ERROR WebSocketServer::Send(const char * msg) +bool WebSocketServer::OnWebSocketMessageReceived(char * msg) { - VerifyOrReturnError(nullptr != gWebSocketInstance, CHIP_ERROR_INCORRECT_STATE); - - chip::Platform::ScopedMemoryBuffer buffer; - VerifyOrReturnError(buffer.Calloc(LWS_PRE + strlen(msg)), CHIP_ERROR_NO_MEMORY); - memcpy(&buffer[LWS_PRE], (void *) msg, strlen(msg)); - lws_write(gWebSocketInstance, &buffer[LWS_PRE], strlen(msg), LWS_WRITE_TEXT); + auto shouldContinue = mDelegate->OnWebSocketMessageReceived(msg); + if (!shouldContinue) + { + mRunning = false; + } + return shouldContinue; +} +CHIP_ERROR WebSocketServer::Send(const char * msg) +{ + std::lock_guard lock(gMutex); + gMessageQueue.push_back(msg); return CHIP_NO_ERROR; } diff --git a/examples/common/websocket-server/WebSocketServer.h b/examples/common/websocket-server/WebSocketServer.h index 712cf185d8f4a0..994f10fcbc1b79 100644 --- a/examples/common/websocket-server/WebSocketServer.h +++ b/examples/common/websocket-server/WebSocketServer.h @@ -23,9 +23,17 @@ #include #include -class WebSocketServer +#include + +class WebSocketServer : public WebSocketServerDelegate { public: CHIP_ERROR Run(chip::Optional port, WebSocketServerDelegate * delegate); CHIP_ERROR Send(const char * msg); + + bool OnWebSocketMessageReceived(char * msg) override; + +private: + bool mRunning; + WebSocketServerDelegate * mDelegate = nullptr; }; diff --git a/examples/placeholder/py_matter_placeholder_adapter/BUILD.gn b/examples/placeholder/py_matter_placeholder_adapter/BUILD.gn new file mode 100644 index 00000000000000..3a5e0802fe4977 --- /dev/null +++ b/examples/placeholder/py_matter_placeholder_adapter/BUILD.gn @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/build.gni") +import("//build_overrides/chip.gni") + +import("//build_overrides/pigweed.gni") +import("$dir_pw_build/python.gni") + +pw_python_package("matter_placeholder_adapter") { + setup = [ + "setup.py", + "setup.cfg", + "pyproject.toml", + ] + + sources = [ + "matter_placeholder_adapter/__init__.py", + "matter_placeholder_adapter/adapter.py", + ] + + # TODO: at a future time consider enabling all (* or missing) here to get + # pylint checking these files + static_analysis = [] +} diff --git a/examples/placeholder/py_matter_placeholder_adapter/matter_placeholder_adapter/__init__.py b/examples/placeholder/py_matter_placeholder_adapter/matter_placeholder_adapter/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/examples/placeholder/py_matter_placeholder_adapter/matter_placeholder_adapter/adapter.py b/examples/placeholder/py_matter_placeholder_adapter/matter_placeholder_adapter/adapter.py new file mode 100644 index 00000000000000..8073282d83683a --- /dev/null +++ b/examples/placeholder/py_matter_placeholder_adapter/matter_placeholder_adapter/adapter.py @@ -0,0 +1,101 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json + +# These constants represent the vocabulary used for the incoming JSON. +_CLUSTER_ID = 'clusterId' +_ENDPOINT_ID = 'endpointId' +_COMMAND_ID = 'commandId' +_ATTRIBUTE_ID = 'attributeId' +_WAIT_TYPE = 'waitType' + +# These constants represent the vocabulary used for the outgoing data. +_CLUSTER = 'cluster' +_ENDPOINT = 'endpoint' +_COMMAND = 'command' +_ATTRIBUTE = 'attribute' +_WAIT_FOR = 'wait_for' + + +class Adapter: + def __init__(self, definitions): + self.encoder = PlaceholderEncoder() + self.decoder = PlaceholderDecoder(definitions) + + def encode(self, request): + return self.encoder.encode(request) + + def decode(self, response): + return self.decoder.decode(response) + + +class PlaceholderEncoder: + def encode(self, request): + cluster = request.cluster + command = request.command + if cluster is None or command is None: + return '' + + return request.command + + +class PlaceholderDecoder: + def __init__(self, definitions): + self.__definitions = definitions + + def decode(self, payload): + payloads, logs = self.__get_payload_content(payload) + json_response = payloads[0] + + decoded_response = {} + + for key, value in json_response.items(): + if key == _CLUSTER_ID: + decoded_response[_CLUSTER] = self.__definitions.get_cluster_name( + value) + elif key == _ENDPOINT_ID: + decoded_response[_ENDPOINT] = value + elif key == _COMMAND_ID: + clusterId = json_response[_CLUSTER_ID] + decoded_response[_COMMAND] = self.__definitions.get_response_name( + clusterId, value) + elif key == _ATTRIBUTE_ID: + clusterId = json_response[_CLUSTER_ID] + decoded_response[_ATTRIBUTE] = self.__definitions.get_attribute_name( + clusterId, value) + elif key == _WAIT_TYPE: + decoded_response[_WAIT_FOR] = value + else: + # Raise an error since the other fields probably needs to be translated too. + raise KeyError(f'Error: field "{key}" in unsupported') + + return decoded_response, logs + + def __get_payload_content(self, payload): + json_payload = json.loads(payload) + logs = list(map(MatterLog, json_payload.get('logs'))) + results = json_payload.get('results') + return results, logs + + +class MatterLog: + def __init__(self, log): + self.module = log['module'] + self.level = log['category'] + + base64_message = log["message"].encode('utf-8') + decoded_message_bytes = base64.b64decode(base64_message) + self.message = decoded_message_bytes.decode('utf-8') diff --git a/examples/placeholder/py_matter_placeholder_adapter/pyproject.toml b/examples/placeholder/py_matter_placeholder_adapter/pyproject.toml new file mode 100644 index 00000000000000..9080987af6e7be --- /dev/null +++ b/examples/placeholder/py_matter_placeholder_adapter/pyproject.toml @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ['setuptools', 'wheel'] +build-backend = 'setuptools.build_meta' diff --git a/examples/placeholder/py_matter_placeholder_adapter/setup.cfg b/examples/placeholder/py_matter_placeholder_adapter/setup.cfg new file mode 100644 index 00000000000000..1775fc5267f8d3 --- /dev/null +++ b/examples/placeholder/py_matter_placeholder_adapter/setup.cfg @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[metadata] +name = matter_placeholder_adapter +version = 0.0.1 +author = Project CHIP Authors +description = placeholder adapter for matter_yamltests + +[options] +packages = find: +zip_safe = False diff --git a/examples/placeholder/py_matter_placeholder_adapter/setup.py b/examples/placeholder/py_matter_placeholder_adapter/setup.py new file mode 100644 index 00000000000000..21ddf9f74cd2fa --- /dev/null +++ b/examples/placeholder/py_matter_placeholder_adapter/setup.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""The matter_placeholder_adapter package.""" + +import setuptools # type: ignore + +setuptools.setup() # Package definition in setup.cfg diff --git a/scripts/py_matter_yamltests/BUILD.gn b/scripts/py_matter_yamltests/BUILD.gn index f0437ce74350fd..b886fb04132c37 100644 --- a/scripts/py_matter_yamltests/BUILD.gn +++ b/scripts/py_matter_yamltests/BUILD.gn @@ -27,11 +27,15 @@ pw_python_package("matter_yamltests") { sources = [ "matter_yamltests/__init__.py", + "matter_yamltests/adapter.py", "matter_yamltests/constraints.py", "matter_yamltests/definitions.py", "matter_yamltests/errors.py", "matter_yamltests/fixes.py", + "matter_yamltests/hooks.py", "matter_yamltests/parser.py", + "matter_yamltests/parser_builder.py", + "matter_yamltests/parser_config.py", "matter_yamltests/pics_checker.py", "matter_yamltests/pseudo_clusters/__init__.py", "matter_yamltests/pseudo_clusters/clusters/delay_commands.py", @@ -39,6 +43,8 @@ pw_python_package("matter_yamltests") { "matter_yamltests/pseudo_clusters/clusters/system_commands.py", "matter_yamltests/pseudo_clusters/pseudo_cluster.py", "matter_yamltests/pseudo_clusters/pseudo_clusters.py", + "matter_yamltests/runner.py", + "matter_yamltests/websocket_runner.py", "matter_yamltests/yaml_loader.py", ] @@ -47,6 +53,7 @@ pw_python_package("matter_yamltests") { tests = [ "test_spec_definitions.py", "test_pics_checker.py", + "test_parser_builder.py", "test_pseudo_clusters.py", "test_yaml_loader.py", ] diff --git a/scripts/py_matter_yamltests/matter_yamltests/adapter.py b/scripts/py_matter_yamltests/matter_yamltests/adapter.py new file mode 100644 index 00000000000000..54f502f6d6a8bf --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/adapter.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod + + +class TestAdapter(ABC): + """ + TestAdapter is an abstract interface that defines the set of methods an adapter + should implement. + + Adapters are used to translate test step requests and test responses back and forth + between the format used by the matter_yamltests package and the format used by the + adapter target. + + Some examples of adapters includes chip-repl, chip-tool and the placeholder applications. + """ + + @abstractmethod + def encode(self, request): + """Encode a test step request from the matter_yamltests format to the adapter format.""" + pass + + @abstractmethod + def decode(self, response): + """ + Decode a test step response from the adapter format to the matter_yamltests format. + + This method returns a tuple containing both the decoded response and additional logs + from the adapter. + """ + pass diff --git a/scripts/py_matter_yamltests/matter_yamltests/hooks.py b/scripts/py_matter_yamltests/matter_yamltests/hooks.py new file mode 100644 index 00000000000000..f471af83408589 --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/hooks.py @@ -0,0 +1,252 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .errors import TestStepError + + +class TestParserHooks(): + def start(self, count: int): + """ + This method is called when the parser starts parsing a set of files. + + Parameters + ---------- + count: int + The number of files that will be parsed. + """ + pass + + def stop(self, duration: int): + """ + This method is called when the parser is done parsing a set of files. + + Parameters + ---------- + duration: int + How long it took to parse the set of files, in milliseconds. + """ + pass + + def test_start(self, name: str): + """ + This method is called when the parser starts parsing a single file. + + Parameters + ---------- + name: str + The name of the file that will be parsed. + """ + pass + + def test_failure(self, exception: TestStepError, duration: int): + """ + This method is called when parsing a single file fails. + + Parameters + ---------- + exception: TestStepError + An exception describing why parsing the file has failed. + duration: int + How long it took to parse the file, in milliseconds. + """ + pass + + def test_success(self, duration: int): + """ + This method is called when parsing a single file succeeds. + + Parameters + ---------- + duration: int + How long it took to parse the file, in milliseconds. + """ + pass + + +class TestRunnerHooks(): + def start(self, count: int): + """ + This method is called when the runner starts running a set of tests. + + Parameters + ---------- + count: int + The number of files that will be run. + """ + pass + + def stop(self, duration: int): + """ + This method is called when the runner is done running a set of tests. + + Parameters + ---------- + duration: int + How long it took to run the set of tests, in milliseconds. + """ + pass + + def test_start(self, name: str, count: int): + """ + This method is called when the runner starts running a single test. + + Parameters + ---------- + name: str + The name of the test that is starting. + + count: int + The number of steps from the test that will be run. + """ + pass + + def test_stop(self, exception: Exception, duration: int): + """ + This method is called when the runner is done running a single test. + + Parameters + ---------- + exception: Exception + An optional exception describing why running the test has failed. + + duration: int + How long it took to run the test, in milliseconds. + """ + pass + + def step_skipped(self, name: str): + """ + This method is called when running a step is skipped. + + Parameters + ---------- + name: str + The name of the test step that is skipped. + """ + pass + + def step_start(self, name: str): + """ + This method is called when the runner starts running a step from the test. + + Parameters + ---------- + name: str + The name of the test step that is starting. + """ + pass + + def step_success(self, logger, logs, duration: int): + """ + This method is called when running a step succeeds. + + Parameters + ---------- + logger: + An object containing details about the different checks against the response. + + logs: + Optional logs from the adapter. + + duration: int + How long it took to run the test step, in milliseconds. + """ + pass + + def step_failure(self, logger, logs, duration: int, expected, received): + """ + This method is called when running a step fails. + + Parameters + ---------- + logger: + An object containing details about the different checks against the response. + + logs: + Optional logs from the adapter. + + duration: int + How long it took to run the test step, in milliseconds. + + expected: + The expected response as defined by the test step. + + received: + The received response. + """ + pass + + def step_unknown(self): + """ + This method is called when the result of running a step is unknown. For example during a dry-run. + """ + pass + + +class WebSocketRunnerHooks(): + def connecting(self, url: str): + """ + This method is called when the websocket is attempting to connect to a remote. + + Parameters + ---------- + url: str + The url the websocket is trying to connect to. + """ + pass + + def abort(self, url: str): + """ + This method is called when the websocket connection fails and will not be retried. + + Parameters + ---------- + url: str + The url the websocket has failed to connect to. + """ + pass + + def success(self, duration: int): + """ + This method is called when the websocket connection is established. + + Parameters + ---------- + duration: int + How long it took to connect since the last retry, in milliseconds. + """ + pass + + def failure(self, duration: int): + """ + This method is called when the websocket connection fails and will be retried. + + Parameters + ---------- + duration: int + How long it took to fail since the last retry, in milliseconds. + """ + pass + + def retry(self, interval_between_retries_in_seconds: int): + """ + This method is called when the websocket connection will be retried in the given interval. + + Parameters + ---------- + interval_between_retries_in_seconds: int + How long we will wait before retrying to connect, in seconds. + """ + pass diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser_builder.py b/scripts/py_matter_yamltests/matter_yamltests/parser_builder.py new file mode 100644 index 00000000000000..ae4458ae6990af --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/parser_builder.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import time +from dataclasses import dataclass, field + +from .hooks import TestParserHooks +from .parser import TestParser, TestParserConfig + + +@dataclass +class TestParserBuilderOptions: + """ + TestParserBuilderOptions allows configuring the behavior of a + TestParserBuilder instance. + + stop_on_error: If set to False the parser will continue parsing + the next test instead of aborting if an error is + encountered while parsing a particular test file. + """ + stop_on_error: bool = True + + +@dataclass +class TestParserBuilderConfig: + """ + TestParserBuilderConfig defines a set of common properties that will be used + by a TestParserBuilder instance for parsing the given list of tests. + + tests: A list of YAML tests to be parsed. + + parser_config: A common configuration for the tests to be parsed. + + options: A common set of options for the tests to be parsed. + + hooks: A configurable set of hooks to be called at various steps during + parsing. It may may allow the callers to gain insights about the + current parsing state. + """ + tests: list[str] = field(default_factory=list) + parser_config: TestParserConfig = TestParserConfig() + hooks: TestParserHooks = TestParserHooks() + options: TestParserBuilderOptions = TestParserBuilderOptions() + + +class TestParserBuilder: + """ + TestParserBuilder is an iterator over a set of tests using a common configuration. + """ + + def __init__(self, config: TestParserBuilderConfig = TestParserBuilderConfig()): + self.__tests = copy.copy(config.tests) + self.__config = config + self.__duration = 0 + self.__done = False + + def __iter__(self): + self.__config.hooks.start(len(self.__tests)) + return self + + def __next__(self): + if len(self.__tests): + return self.__get_test_parser(self.__tests.pop(0)) + + if not self.__done: + self.__config.hooks.stop(round(self.__duration)) + self.__done = True + + raise StopIteration + + def __get_test_parser(self, test_file: str) -> TestParser: + start = time.time() + + parser = None + exception = None + try: + self.__config.hooks.test_start(test_file) + parser = TestParser(test_file, self.__config.parser_config) + except Exception as e: + exception = e + + duration = round((time.time() - start) * 1000, 0) + self.__duration += duration + if exception: + self.__config.hooks.test_failure(exception, duration) + if self.__config.options.stop_on_error: + raise StopIteration + return None + + self.__config.hooks.test_success(duration) + return parser diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser_config.py b/scripts/py_matter_yamltests/matter_yamltests/parser_config.py new file mode 100644 index 00000000000000..251aa5ebb17266 --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/parser_config.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .yaml_loader import YamlLoader + + +class TestConfigParser: + """ + TestConfigParser exposes a single static method for reading the config section + of a given YAML test file as a dictionary. + + For example a command line tool can use it to dynamically add extra options + but using test specific options in addition to the generic options. + """ + + def get_config(test_file: str): + config_options = {} + + yaml_loader = YamlLoader() + _, _, config, _ = yaml_loader.load(test_file) + config_options = {key: value if not isinstance( + value, dict) else value['defaultValue'] for key, value in config.items()} + return config_options diff --git a/scripts/py_matter_yamltests/matter_yamltests/runner.py b/scripts/py_matter_yamltests/matter_yamltests/runner.py new file mode 100644 index 00000000000000..66855a528bc4a7 --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/runner.py @@ -0,0 +1,207 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass + +from .adapter import TestAdapter +from .hooks import TestRunnerHooks +from .parser import TestParser +from .parser_builder import TestParserBuilder, TestParserBuilderConfig +from .pseudo_clusters.pseudo_clusters import PseudoClusters + + +@dataclass +class TestRunnerOptions: + """ + TestRunnerOptions allows configuring the behavior of a TestRunner + instance. + + stop_on_error: If set to False the runner will continue executing + the next test step instead of aborting if an error + is encountered while running a particular test file. + + stop_on_warning: If set to true the runner will stop running the + tests if a warning is encountered while running + a particular test file. + + stop_at_number: If set to any non negative number, the runner will + stop running the tests if a step index matches + the number. This is mostly useful when running + a single test file and for debugging purposes. + """ + stop_on_error: bool = True + stop_on_warning: bool = False + stop_at_number: int = -1 + + +@dataclass +class TestRunnerConfig: + """ + TestRunnerConfig defines a set of common properties that will be used + by a TestRunner instance for running the given set of tests. + + adapter: An adapter to use to translate from the matter_yamltests format + to the target format. + + pseudo_clusters: The pseudo clusters to use while running tests. + + options: A common set of options for the tests to be runned. + + hooks: A configurable set of hooks to be called at various steps while + running. It may may allow the callers to gain insights about the + current running state. + """ + adapter: TestAdapter = None + pseudo_clusters: PseudoClusters = PseudoClusters([]) + options: TestRunnerOptions = TestRunnerOptions() + hooks: TestRunnerHooks = TestRunnerHooks() + + +class TestRunnerBase(ABC): + """ + TestRunnerBase is an abstract interface that defines the set of methods a runner + should implement. + + A runner is responsible for executing a test step. + """ + @abstractmethod + async def start(self): + """ + This method is called before running the steps of a particular test file. + It may allow the runner to perform some setup tasks. + """ + pass + + @abstractmethod + async def stop(self): + """ + This method is called after running the steps of a particular test file. + It may allow the runner to perform some cleanup tasks. + """ + pass + + @abstractmethod + async def execute(self, request): + """ + This method executes a request using the adapter format, and returns a response + using the adapter format. + """ + pass + + @abstractmethod + def run(self, config: TestRunnerConfig) -> bool: + """ + This method runs a test suite. + + Returns + ------- + bool + A boolean indicating if the run has succeeded. + """ + pass + + +class TestRunner(TestRunnerBase): + """ + TestRunner is a default runner implementation. + """ + async def start(self): + return + + async def execute(self, request): + return request + + async def stop(self): + return + + def run(self, parser_builder_config: TestParserBuilderConfig, runner_config: TestRunnerConfig): + if runner_config and runner_config.hooks: + start = time.time() + runner_config.hooks.start(len(parser_builder_config.tests)) + + parser_builder = TestParserBuilder(parser_builder_config) + for parser in parser_builder: + if not parser or not runner_config: + continue + + result = asyncio.run(self._run(parser, runner_config)) + if isinstance(result, Exception): + raise (result) + elif not result: + return False + + if runner_config and runner_config.hooks: + duration = round((time.time() - start) * 1000) + runner_config.hooks.stop(duration) + + return True + + async def _run(self, parser: TestParser, config: TestRunnerConfig): + status = True + try: + await self.start() + + hooks = config.hooks + hooks.test_start(parser.name, parser.tests.count) + + test_duration = 0 + for idx, request in enumerate(parser.tests): + if not request.is_pics_enabled: + hooks.step_skipped(request.label) + continue + elif not config.adapter: + hooks.step_start(request.label) + hooks.step_unknown() + continue + else: + hooks.step_start(request.label) + + start = time.time() + if config.pseudo_clusters.supports(request): + responses, logs = await config.pseudo_clusters.execute(request) + else: + responses, logs = config.adapter.decode(await self.execute(config.adapter.encode(request))) + duration = round((time.time() - start) * 1000, 2) + test_duration += duration + + logger = request.post_process_response(responses) + + if logger.is_failure(): + hooks.step_failure(logger, logs, duration, + request.responses, responses) + else: + hooks.step_success(logger, logs, duration) + + if logger.is_failure() and config.options.stop_on_error: + status = False + break + + if logger.warnings and config.options.stop_on_warning: + status = False + break + + if (idx + 1) == config.options.stop_at_number: + break + + hooks.test_stop(round(test_duration)) + + except Exception as exception: + status = exception + finally: + await self.stop() + return status diff --git a/scripts/py_matter_yamltests/matter_yamltests/websocket_runner.py b/scripts/py_matter_yamltests/matter_yamltests/websocket_runner.py new file mode 100644 index 00000000000000..2045bba732be13 --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/websocket_runner.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import time +from dataclasses import dataclass + +import websockets + +from .hooks import WebSocketRunnerHooks +from .runner import TestRunner + + +@dataclass +class WebSocketRunnerConfig: + server_address: str = 'localhost' + server_port: int = '9002' + server_path: str = None + server_arguments: str = None + hooks: WebSocketRunnerHooks = WebSocketRunnerHooks() + + +class WebSocketRunner(TestRunner): + def __init__(self, config: WebSocketRunnerConfig): + self._client = None + self._server = None + self._hooks = config.hooks + + self._server_connection_url = self._make_server_connection_url( + config.server_address, config.server_port) + self._server_startup_command = self._make_server_startup_command( + config.server_path, config.server_arguments, config.server_port) + + async def start(self): + self._server = await self._start_server(self._server_startup_command) + self._client = await self._start_client(self._server_connection_url) + + async def stop(self): + await self._stop_client(self._client) + await self._stop_server(self._server) + self._client = None + self._server = None + + async def execute(self, request): + instance = self._client + if instance: + await instance.send(request) + return await instance.recv() + return None + + async def _start_client(self, url, max_retries=4, interval_between_retries=1): + if max_retries: + start = time.time() + try: + self._hooks.connecting(url) + connection = await websockets.connect(url) + duration = round((time.time() - start) * 1000, 0) + self._hooks.success(duration) + return connection + except Exception: + duration = round((time.time() - start) * 1000, 0) + self._hooks.failure(duration) + self._hooks.retry(interval_between_retries) + time.sleep(interval_between_retries) + return await self._start_client(url, max_retries - 1, interval_between_retries + 1) + + self._hooks.abort(url) + raise Exception(f'Connecting to {url} failed.') + + async def _stop_client(self, instance): + if instance: + await instance.close() + + async def _start_server(self, command): + instance = None + if command: + instance = subprocess.Popen(command, stdout=subprocess.DEVNULL) + return instance + + async def _stop_server(self, instance): + if instance: + instance.kill() + + def _make_server_connection_url(self, address: str, port: int): + return 'ws://' + address + ':' + str(port) + + def _make_server_startup_command(self, path: str, arguments: str, port: int): + if path is None: + return None + elif arguments is None: + return [path] + ['--port', str(port)] + else: + return [path] + [arg.strip() for arg in arguments.split(' ')] + ['--port', str(port)] diff --git a/scripts/py_matter_yamltests/test_parser_builder.py b/scripts/py_matter_yamltests/test_parser_builder.py new file mode 100644 index 00000000000000..186dfad98f0def --- /dev/null +++ b/scripts/py_matter_yamltests/test_parser_builder.py @@ -0,0 +1,207 @@ +#!/usr/bin/env -S python3 -B +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import mock_open, patch + +from matter_yamltests.hooks import TestParserHooks +from matter_yamltests.parser import TestParser +from matter_yamltests.parser_builder import TestParserBuilder, TestParserBuilderConfig + +simple_yaml = ''' +name: Hello World +''' + +valid_yaml = ''' +name: TestOnOff + +tests: + - label: "Toggle the light" + cluster: "OnOff" + command: "Toggle" + + - label: "Toggle the light" + cluster: "OnOff" + command: "Toggle" +''' + +invalid_yaml = ''' +name: TestOnOff + +tests: + - label: "Toggle the light" + cluster: "OnOff" + command: "Toggle" + + - label: "Toggle the light" + cluster: "OnOff" + command: "Toggle" + + - label: "Toggle the light" + cluster: "OnOff" + command: "Toggle" + + - label: "Toggle the light" + cluster_wrong_key: "OnOff" + command: "Toggle" + + - label: "Toggle the light" + cluster: "OnOff" + command: "Toggle" +''' + + +class TestHooks(TestParserHooks): + def __init__(self): + self.start_count = 0 + self.stop_count = 0 + self.test_start_count = 0 + self.test_failure_count = 0 + self.test_success_count = 0 + + def start(self, count): + self.start_count += 1 + + def stop(self, duration): + self.stop_count += 1 + + def test_start(self, name): + self.test_start_count += 1 + + def test_success(self, duration): + self.test_success_count += 1 + + def test_failure(self, exception, duration): + self.test_failure_count += 1 + + +def mock_open_with_parameter_content(content): + file_object = mock_open(read_data=content).return_value + file_object.__iter__.return_value = content.splitlines(True) + return file_object + + +@patch('builtins.open', new=mock_open_with_parameter_content) +class TestSuiteParserBuilder(unittest.TestCase): + def test_parser_builder_config_defaults(self): + parser_builder_config = TestParserBuilderConfig() + self.assertIsInstance(parser_builder_config, TestParserBuilderConfig) + self.assertIsNotNone(parser_builder_config.tests) + self.assertIsNotNone(parser_builder_config.parser_config) + self.assertIsNotNone(parser_builder_config.options) + self.assertIsNotNone(parser_builder_config.hooks) + + def test_parser_builder_config_with_tests(self): + tests = [simple_yaml, simple_yaml] + parser_builder_config = TestParserBuilderConfig(tests) + self.assertIsInstance(parser_builder_config, TestParserBuilderConfig) + self.assertIsNotNone(parser_builder_config.tests) + self.assertEqual(len(tests), len(parser_builder_config.tests)) + self.assertIsNotNone(parser_builder_config.parser_config) + self.assertIsNotNone(parser_builder_config.options) + + def test_parser_builder_default(self): + parser_builder = TestParserBuilder() + self.assertIsInstance(parser_builder, TestParserBuilder) + self.assertRaises(StopIteration, next, parser_builder) + + def test_parser_builder_with_empty_config(self): + parser_builder_config = TestParserBuilderConfig() + parser_builder = TestParserBuilder(parser_builder_config) + self.assertIsInstance(parser_builder, TestParserBuilder) + self.assertRaises(StopIteration, next, parser_builder) + + def test_parser_builder_with_a_single_test(self): + tests = [valid_yaml] + + parser_builder_config = TestParserBuilderConfig(tests) + parser_builder = TestParserBuilder(parser_builder_config) + self.assertIsInstance(parser_builder, TestParserBuilder) + self.assertIsInstance(next(parser_builder), TestParser) + self.assertRaises(StopIteration, next, parser_builder) + + def test_parser_builder_with_a_multiple_tests(self): + tests = [valid_yaml] * 5 + + parser_builder_config = TestParserBuilderConfig(tests) + parser_builder = TestParserBuilder(parser_builder_config) + self.assertIsInstance(parser_builder, TestParserBuilder) + + for i in range(0, 5): + self.assertIsInstance(next(parser_builder), TestParser) + + self.assertRaises(StopIteration, next, parser_builder) + + def test_parser_builder_config_hooks_single_test_with_multiple_steps(self): + tests = [valid_yaml] + hooks = TestHooks() + parser_builder_config = TestParserBuilderConfig(tests, hooks=hooks) + + parser_builder = TestParserBuilder(parser_builder_config) + self.assertIsInstance(parser_builder, TestParserBuilder) + + for parser in parser_builder: + pass + + self.assertRaises(StopIteration, next, parser_builder) + + self.assertEqual(hooks.start_count, 1) + self.assertEqual(hooks.stop_count, 1) + self.assertEqual(hooks.test_start_count, 1) + self.assertEqual(hooks.test_success_count, 1) + self.assertEqual(hooks.test_failure_count, 0) + + def test_parser_builder_config_hooks_multiple_test_with_multiple_steps(self): + tests = [valid_yaml] * 5 + hooks = TestHooks() + parser_builder_config = TestParserBuilderConfig(tests, hooks=hooks) + + parser_builder = TestParserBuilder(parser_builder_config) + self.assertIsInstance(parser_builder, TestParserBuilder) + + for parser in parser_builder: + pass + + self.assertRaises(StopIteration, next, parser_builder) + + self.assertEqual(hooks.start_count, 1) + self.assertEqual(hooks.stop_count, 1) + self.assertEqual(hooks.test_start_count, 5) + self.assertEqual(hooks.test_success_count, 5) + self.assertEqual(hooks.test_failure_count, 0) + + def test_parser_builder_config_with_errors(self): + tests = [invalid_yaml] + hooks = TestHooks() + parser_builder_config = TestParserBuilderConfig(tests, hooks=hooks) + + parser_builder = TestParserBuilder(parser_builder_config) + self.assertIsInstance(parser_builder, TestParserBuilder) + + for parser in parser_builder: + pass + + self.assertRaises(StopIteration, next, parser_builder) + + self.assertEqual(hooks.start_count, 1) + self.assertEqual(hooks.stop_count, 1) + self.assertEqual(hooks.test_start_count, 1) + self.assertEqual(hooks.test_success_count, 0) + self.assertEqual(hooks.test_failure_count, 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/setup/requirements.txt b/scripts/setup/requirements.txt index 6a0ba5162d2ff0..5cc364d49140d8 100644 --- a/scripts/setup/requirements.txt +++ b/scripts/setup/requirements.txt @@ -84,3 +84,4 @@ tornado # YAML test harness diskcache +websockets diff --git a/scripts/tests/chiptest/linux.py b/scripts/tests/chiptest/linux.py index 1b72819d4992e4..73bbb93ca45050 100644 --- a/scripts/tests/chiptest/linux.py +++ b/scripts/tests/chiptest/linux.py @@ -183,4 +183,5 @@ def PathsWithNetworkNamespaces(paths: ApplicationPaths) -> ApplicationPaths: tv_app='ip netns exec app'.split() + paths.tv_app, bridge_app='ip netns exec app'.split() + paths.bridge_app, chip_repl_yaml_tester_cmd='ip netns exec tool'.split() + paths.chip_repl_yaml_tester_cmd, + chip_tool_with_python_cmd='ip netns exec tool'.split() + paths.chip_tool_with_python_cmd, ) diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py index 52238719e7a211..b9842647105ab5 100644 --- a/scripts/tests/chiptest/test_definition.py +++ b/scripts/tests/chiptest/test_definition.py @@ -166,9 +166,10 @@ class ApplicationPaths: tv_app: typing.List[str] bridge_app: typing.List[str] chip_repl_yaml_tester_cmd: typing.List[str] + chip_tool_with_python_cmd: typing.List[str] def items(self): - return [self.chip_tool, self.all_clusters_app, self.lock_app, self.ota_provider_app, self.ota_requestor_app, self.tv_app, self.bridge_app, self.chip_repl_yaml_tester_cmd] + return [self.chip_tool, self.all_clusters_app, self.lock_app, self.ota_provider_app, self.ota_requestor_app, self.tv_app, self.bridge_app, self.chip_repl_yaml_tester_cmd, self.chip_tool_with_python_cmd] @dataclass @@ -225,7 +226,8 @@ def to_s(self): class TestRunTime(Enum): CHIP_TOOL_BUILTIN = auto() # run via chip-tool built-in test commands - PYTHON_YAML = auto() # use the python yaml test runner + CHIP_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool + CHIP_REPL_PYTHON = auto() # use the python yaml test runner @dataclass @@ -274,7 +276,7 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, ti for path in paths.items(): # Do not add chip-tool or chip-repl-yaml-tester-cmd to the register - if path == paths.chip_tool or path == paths.chip_repl_yaml_tester_cmd: + if path == paths.chip_tool or path == paths.chip_repl_yaml_tester_cmd or path == paths.chip_tool_with_python_cmd: continue # For the app indicated by self.target, give it the 'default' key to add to the register @@ -291,7 +293,7 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, ti # so it will be commissionable again. app.factoryReset() - tool_cmd = paths.chip_tool + tool_cmd = paths.chip_tool if test_runtime != TestRunTime.CHIP_TOOL_PYTHON else paths.chip_tool_with_python_cmd files_to_unlink = [ '/tmp/chip_tool_config.ini', @@ -309,11 +311,14 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str, ti app.start() pairing_cmd = tool_cmd + ['pairing', 'code', TEST_NODE_ID, app.setupCode] test_cmd = tool_cmd + ['tests', self.run_name] + ['--PICS', pics_file] + if test_runtime == TestRunTime.CHIP_TOOL_PYTHON: + pairing_cmd += ['--server_path'] + [paths.chip_tool[-1]] + test_cmd += ['--server_path'] + [paths.chip_tool[-1]] if dry_run: logging.info(" ".join(pairing_cmd)) logging.info(" ".join(test_cmd)) - elif test_runtime == TestRunTime.PYTHON_YAML: + elif test_runtime == TestRunTime.CHIP_REPL_PYTHON: chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd python_cmd = chip_repl_yaml_tester_cmd + \ ['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file] diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py index d4963f1623d1fd..357148da8a35b5 100755 --- a/scripts/tests/run_test_suite.py +++ b/scripts/tests/run_test_suite.py @@ -146,23 +146,29 @@ class RunContext: help='What test tags to exclude when running. Exclude options takes precedence over include.', ) @click.option( - '--run-yamltests-with-chip-repl', - default=False, - is_flag=True, - help='Run YAML tests using chip-repl based python parser only') + '--runner', + type=click.Choice(['codegen', 'chip_repl_python', 'chip_tool_python'], case_sensitive=False), + default='codegen', + help='Run YAML tests using the specified runner.') @click.option( '--chip-tool', help='Binary path of chip tool app to use to run the test') @click.pass_context def main(context, dry_run, log_level, target, target_glob, target_skip_glob, - no_log_timestamps, root, internal_inside_unshare, include_tags, exclude_tags, run_yamltests_with_chip_repl, chip_tool): + no_log_timestamps, root, internal_inside_unshare, include_tags, exclude_tags, runner, chip_tool): # Ensures somewhat pretty logging of what is going on log_fmt = '%(asctime)s.%(msecs)03d %(levelname)-7s %(message)s' if no_log_timestamps: log_fmt = '%(levelname)-7s %(message)s' coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt) - if chip_tool is None and not run_yamltests_with_chip_repl: + runtime = TestRunTime.CHIP_TOOL_BUILTIN + if runner == 'chip_repl_python': + runtime = TestRunTime.CHIP_REPL_PYTHON + elif runner == 'chip_tool_python': + runtime = TestRunTime.CHIP_TOOL_PYTHON + + if chip_tool is None and not runtime == TestRunTime.CHIP_REPL_PYTHON: # non yaml tests REQUIRE chip-tool. Yaml tests should not require chip-tool chip_tool = FindBinaryPath('chip-tool') @@ -173,7 +179,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, exclude_tags = set([TestTag.__members__[t] for t in exclude_tags]) # Figures out selected test that match the given name(s) - if run_yamltests_with_chip_repl: + if runtime == TestRunTime.CHIP_REPL_PYTHON: all_tests = [test for test in chiptest.AllYamlTests()] else: all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)] @@ -218,7 +224,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob, context.obj = RunContext(root=root, tests=tests, in_unshare=internal_inside_unshare, chip_tool=chip_tool, dry_run=dry_run, - runtime=TestRunTime.PYTHON_YAML if run_yamltests_with_chip_repl else TestRunTime.CHIP_TOOL_BUILTIN, + runtime=runtime, include_tags=include_tags, exclude_tags=exclude_tags) @@ -262,6 +268,9 @@ def cmd_list(context): @click.option( '--chip-repl-yaml-tester', help='what python script to use for running yaml tests using chip-repl as controller') +@click.option( + '--chip-tool-with-python', + help='what python script to use for running yaml tests using chip-tool as controller') @click.option( '--pics-file', type=click.Path(exists=True), @@ -280,7 +289,7 @@ def cmd_list(context): type=int, help='If provided, fail if a test runs for longer than this time') @click.pass_context -def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, ota_requestor_app, tv_app, bridge_app, chip_repl_yaml_tester, pics_file, keep_going, test_timeout_seconds): +def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, ota_requestor_app, tv_app, bridge_app, chip_repl_yaml_tester, chip_tool_with_python, pics_file, keep_going, test_timeout_seconds): runner = chiptest.runner.Runner() if all_clusters_app is None: @@ -304,6 +313,9 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o if chip_repl_yaml_tester is None: chip_repl_yaml_tester = FindBinaryPath('yamltest_with_chip_repl_tester.py') + if chip_tool_with_python is None: + chip_tool_with_python = FindBinaryPath('chiptool.py') + # Command execution requires an array paths = chiptest.ApplicationPaths( chip_tool=[context.obj.chip_tool], @@ -313,7 +325,8 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o ota_requestor_app=[ota_requestor_app], tv_app=[tv_app], bridge_app=[bridge_app], - chip_repl_yaml_tester_cmd=['python3'] + [chip_repl_yaml_tester] + chip_repl_yaml_tester_cmd=['python3'] + [chip_repl_yaml_tester], + chip_tool_with_python_cmd=['python3'] + [chip_tool_with_python], ) if sys.platform == 'linux': diff --git a/scripts/tests/yaml/__init__.py b/scripts/tests/yaml/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/scripts/tests/yaml/chiptool.py b/scripts/tests/yaml/chiptool.py new file mode 100755 index 00000000000000..9218832ea8c0f5 --- /dev/null +++ b/scripts/tests/yaml/chiptool.py @@ -0,0 +1,102 @@ +#!/usr/bin/env -S python3 -B + +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import relative_importer # isort: split # noqa: F401 + +import asyncio +import json +import sys + +import click +from matter_chip_tool_adapter.decoder import MatterLog +from matter_yamltests.websocket_runner import WebSocketRunner, WebSocketRunnerConfig +from paths_finder import PathsFinder +from runner import CONTEXT_SETTINGS, chiptool, runner_base +from tests_logger import TestColoredLogPrinter, WebSocketRunnerLogger + + +@click.pass_context +def send_yaml_command(ctx, test_name: str, server_path: str, server_arguments: str, pics: str): + parser_builder_config = ctx.invoke(runner_base, test_name=test_name, pics=pics) + + del ctx.params['commands'] + del ctx.params['pics'] + return ctx.forward(chiptool, parser_builder_config) + + +def send_raw_command(command: str, server_path: str, server_arguments: str): + websocket_runner_hooks = WebSocketRunnerLogger() + websocket_runner_config = WebSocketRunnerConfig( + server_path=server_path, server_arguments=server_arguments, hooks=websocket_runner_hooks) + runner = WebSocketRunner(websocket_runner_config) + + async def send_over_websocket(): + payload = None + try: + await runner.start() + payload = await runner.execute(command) + finally: + await runner.stop() + return payload + + payload = asyncio.run(send_over_websocket()) + json_payload = json.loads(payload) + + log_printer = TestColoredLogPrinter() + log_printer.print(MatterLog.decode_logs(json_payload.get('logs'))) + + success = not bool(len([lambda x: x.get('error') for x in json_payload.get('results')])) + return success + + +_DEFAULT_PICS_FILE = 'src/app/tests/suites/certification/ci-pics-values' + + +def chiptool_runner_options(f): + f = click.option('--server_path', type=click.Path(exists=True), default=None, + help='Path to an websocket server to run at launch.')(f) + f = click.option('--server_name', type=str, default='chip-tool', + help='Name of a websocket server to run at launch.')(f) + f = click.option('--server_arguments', type=str, default='interactive server', + help='Optional arguments to pass to the websocket server at launch.')(f) + f = click.option('--PICS', type=click.Path(exists=True), show_default=True, default=_DEFAULT_PICS_FILE, + help='Path to the PICS file to use.')(f) + return f + + +CONTEXT_SETTINGS['ignore_unknown_options'] = True + + +@click.command(context_settings=CONTEXT_SETTINGS) +@click.argument('commands', nargs=-1) +@chiptool_runner_options +@click.pass_context +def chiptool_py(ctx, commands: list[str], server_path: str, server_name: str, server_arguments: str, pics: str): + success = False + + if len(commands) > 1 and commands[0] == 'tests': + success = send_yaml_command(commands[1], server_path, server_arguments, pics) + else: + if server_path is None and server_name: + paths_finder = PathsFinder() + server_path = paths_finder.get(server_name) + success = send_raw_command(' '.join(commands), server_path, server_arguments) + + sys.exit(0 if success else 1) + + +if __name__ == '__main__': + chiptool_py() diff --git a/scripts/tests/yaml/paths_finder.py b/scripts/tests/yaml/paths_finder.py new file mode 100755 index 00000000000000..2f1cb9d6e2b235 --- /dev/null +++ b/scripts/tests/yaml/paths_finder.py @@ -0,0 +1,101 @@ +#!/usr/bin/env -S python3 -B +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import tempfile +from pathlib import Path + +import click +from diskcache import Cache + +_PATHS_CACHE_NAME = 'yaml_runner_cache' +_PATHS_CACHE = Cache(os.path.join(tempfile.gettempdir(), _PATHS_CACHE_NAME)) + +DEFAULT_CHIP_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) + + +class PathsFinder: + def __init__(self, root_dir: str = DEFAULT_CHIP_ROOT): + self.__root_dir = root_dir + + def get(self, target_name: str) -> str: + path = _PATHS_CACHE.get(target_name) + if path and Path(path).is_file(): + return path + + if path: + del _PATHS_CACHE[target_name] + + for path in Path(self.__root_dir).rglob(target_name): + if not path.is_file() or path.name != target_name: + continue + + _PATHS_CACHE[target_name] = str(path) + return str(path) + + return None + + +@click.group() +def finder(): + pass + + +@finder.command() +def view(): + """View the cache entries.""" + for name in _PATHS_CACHE: + print(click.style(f'{name}', bold=True) + f':\t{_PATHS_CACHE[name]}') + + +@finder.command() +@click.argument('key', type=str) +@click.argument('value', type=str) +def add(key: str, value: str): + """Add a cache entry.""" + _PATHS_CACHE[key] = value + + +@finder.command() +@click.argument('name', type=str) +def delete(name: str): + """Delete a cache entry.""" + if name in _PATHS_CACHE: + del _PATHS_CACHE[name] + + +@finder.command() +def reset(): + """Delete all cache entries.""" + for name in _PATHS_CACHE: + del _PATHS_CACHE[name] + + +@finder.command() +@click.argument('name', type=str) +def search(name: str): + """Search for a target and add it to the cache.""" + paths_finder = PathsFinder() + path = paths_finder.get(name) + if path: + print(f'The target "{name}" has been added with the value "{path}".') + else: + print(f'The target "{name}" was not found.') + + +if __name__ == '__main__': + finder() diff --git a/scripts/tests/yaml/relative_importer.py b/scripts/tests/yaml/relative_importer.py new file mode 100644 index 00000000000000..bf1168a8349f86 --- /dev/null +++ b/scripts/tests/yaml/relative_importer.py @@ -0,0 +1,43 @@ +#!/usr/bin/env -S python3 -B + +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +DEFAULT_CHIP_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) +SCRIPT_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'scripts') +EXAMPLES_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'examples') + +try: + import matter_idl # noqa: F401 +except ModuleNotFoundError: + sys.path.append(os.path.join(SCRIPT_PATH, 'py_matter_idl')) + +try: + import matter_yamltests # noqa: F401 +except ModuleNotFoundError: + sys.path.append(os.path.join(SCRIPT_PATH, 'py_matter_yamltests')) + +try: + import matter_chip_tool_adapter # noqa: F401 +except ModuleNotFoundError: + sys.path.append(os.path.join(EXAMPLES_PATH, 'chip-tool', 'py_matter_chip_tool_adapter')) + +try: + import matter_placeholder_adapter # noqa: F401 +except ModuleNotFoundError: + sys.path.append(os.path.join(EXAMPLES_PATH, 'placeholder', 'py_matter_placeholder_adapter')) diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/yaml/runner.py new file mode 100755 index 00000000000000..ec1ae340027ebb --- /dev/null +++ b/scripts/tests/yaml/runner.py @@ -0,0 +1,324 @@ +#!/usr/bin/env -S python3 -B + +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import relative_importer # isort: split # noqa: F401 + +import sys +import traceback +from dataclasses import dataclass + +import click +from matter_yamltests.definitions import SpecDefinitionsFromPaths +from matter_yamltests.parser import TestParserConfig +from matter_yamltests.parser_builder import TestParserBuilderConfig +from matter_yamltests.parser_config import TestConfigParser +from matter_yamltests.pseudo_clusters.pseudo_clusters import PseudoClusters, get_default_pseudo_clusters +from matter_yamltests.runner import TestRunner, TestRunnerConfig, TestRunnerOptions +from matter_yamltests.websocket_runner import WebSocketRunner, WebSocketRunnerConfig +from paths_finder import PathsFinder +from tests_finder import TestsFinder +from tests_logger import TestParserLogger, TestRunnerLogger, WebSocketRunnerLogger + +# +# Options +# + +_DEFAULT_CONFIG_NAME = TestsFinder.get_default_configuration_name() +_DEFAULT_CONFIG_DIR = TestsFinder.get_default_configuration_directory() +_DEFAULT_SPECIFICATIONS_DIR = 'src/app/zap-templates/zcl/data-model/chip/*.xml' +_DEFAULT_PICS_FILE = 'src/app/tests/suites/certification/ci-pics-values' + + +def test_parser_options(f): + f = click.option('--configuration_name', type=str, show_default=True, default=_DEFAULT_CONFIG_NAME, + help='Name of the collection configuration json file to use.')(f) + f = click.option('--configuration_directory', type=click.Path(exists=True), show_default=True, default=_DEFAULT_CONFIG_DIR, + help='Path to the directory containing the tests configuration.')(f) + f = click.option('--specifications_paths', type=click.Path(), show_default=True, default=_DEFAULT_SPECIFICATIONS_DIR, + help='Path to a set of files containing clusters definitions.')(f) + f = click.option('--PICS', type=click.Path(exists=True), show_default=True, default=_DEFAULT_PICS_FILE, + help='Path to the PICS file to use.')(f) + f = click.option('--stop_on_error', type=bool, show_default=True, default=True, + help='Stop parsing on first error.')(f) + f = click.option('--use_default_pseudo_clusters', type=bool, show_default=True, default=True, + help='If enable this option use the set of default clusters provided by the matter_yamltests package.')(f) + return f + + +def test_runner_options(f): + f = click.option('--adapter', type=str, default=None, required=True, show_default=True, + help='The adapter to run the test with.')(f) + f = click.option('--stop_on_error', type=bool, default=True, show_default=True, + help='Stop the test suite on first error.')(f) + f = click.option('--stop_on_warning', type=bool, default=False, show_default=True, + help='Stop the test suite on first warning.')(f) + f = click.option('--stop_at_number', type=int, default=-1, show_default=True, + help='Stop the the test suite at the specified test number.')(f) + f = click.option('--show_adapter_logs', type=bool, default=False, show_default=True, + help='Show additional logs provided by the adapter.')(f) + f = click.option('--show_adapter_logs_on_error', type=bool, default=True, show_default=True, + help='Show additional logs provided by the adapter on error.')(f) + return f + + +def websocket_runner_options(f): + f = click.option('--server_address', type=str, default='localhost', show_default=True, + help='The websocket server address to connect to.')(f) + f = click.option('--server_port', type=int, default=9002, show_default=True, + help='The websocket server port to connect to.')(f) + f = click.option('--server_name', type=str, default=None, + help='Name of a websocket server to run at launch.')(f) + f = click.option('--server_path', type=click.Path(exists=True), default=None, + help='Path to a websocket server to run at launch.')(f) + f = click.option('--server_arguments', type=str, default=None, + help='Optional arguments to pass to the websocket server at launch.')(f) + return f + + +@dataclass +class ParserGroup: + builder_config: TestParserBuilderConfig + pseudo_clusters: PseudoClusters + + +pass_parser_group = click.make_pass_decorator(ParserGroup) + + +# YAML test file contains configurable options defined in their config section. +# +# Those options are test specific and as such can not be listed directly here but +# instead are retrieved from the target test file (if there is a single file) and +# are exposed to click dynamically. +# +# The following code extracts those and, to make it easy for the user to see +# which options are available, list them in a custom section when --help +# is invoked. + +class YamlTestParserGroup(click.Group): + def format_options(self, ctx, formatter): + """Writes all the options into the formatter if they exist.""" + if ctx.custom_options: + params_copy = self.params + non_custom_params = list(filter(lambda x: x.name not in ctx.custom_options, self.params)) + custom_params = list(filter(lambda x: x.name in ctx.custom_options, self.params)) + + self.params = non_custom_params + super().format_options(ctx, formatter) + self.params = params_copy + + opts = [] + for param in custom_params: + rv = param.get_help_record(ctx) + if rv is not None: + opts.append(rv) + + if opts: + custom_section_title = ctx.params.get('test_name') + ' Options' + with formatter.section(custom_section_title): + formatter.write_dl(opts) + else: + super().format_options(ctx, formatter) + + def parse_args(self, ctx, args): + # Run the parser on the supported arguments first in order to get a + # the necessary informations to get read the test file and add + # the potential additional arguments. + supported_args = self.__remove_custom_args(ctx, args) + super().parse_args(ctx, supported_args) + + # Add the potential new arguments to the list of supported params and + # run the parser a new time to read those. + self.__add_custom_params(ctx) + return super().parse_args(ctx, args) + + def __remove_custom_args(self, ctx, args): + # Remove all the unsupported options from the command line string. + params_name = [param.name for param in self.params] + + supported_args = [] + skipArgument = False + for arg in args: + if arg.startswith('--') and arg not in params_name: + skipArgument = True + continue + if skipArgument: + skipArgument = False + continue + supported_args.append(arg) + + return supported_args + + def __add_custom_params(self, ctx): + tests_finder = TestsFinder(ctx.params.get('configuration_directory'), ctx.params.get('configuration_name')) + tests = tests_finder.get(ctx.params.get('test_name')) + + custom_options = {} + + # There is a single test, extract the custom config + if len(tests) == 1: + try: + custom_options = TestConfigParser.get_config(tests[0]) + except Exception: + pass + for key, value in custom_options.items(): + param = click.Option(['--' + key], default=value, show_default=True) + # click converts parameter name to lowercase internally, so we need to override + # this behavior in order to override the correct key. + param.name = key + self.params.append(param) + + ctx.custom_options = custom_options + + +CONTEXT_SETTINGS = dict( + default_map={ + 'chiptool': { + 'adapter': 'matter_chip_tool_adapter.adapter', + 'server_name': 'chip-tool', + 'server_arguments': 'interactive server', + }, + 'app1': { + 'configuration_directory': 'examples/placeholder/linux/apps/app1', + 'adapter': 'matter_placeholder_adapter.adapter', + 'server_name': 'chip-app1', + 'server_arguments': '--interactive', + }, + 'app2': { + 'configuration_directory': 'examples/placeholder/linux/apps/app2', + 'adapter': 'matter_placeholder_adapter.adapter', + 'server_name': 'chip-app2', + 'server_arguments': '--interactive', + }, + }, + max_content_width=120, +) + + +@click.group(cls=YamlTestParserGroup, context_settings=CONTEXT_SETTINGS) +@click.argument('test_name') +@test_parser_options +@click.pass_context +def runner_base(ctx, configuration_directory: str, test_name: str, configuration_name: str, pics: str, specifications_paths: str, stop_on_error: bool, use_default_pseudo_clusters: bool, **kwargs): + pseudo_clusters = get_default_pseudo_clusters() if use_default_pseudo_clusters else PseudoClusters([]) + specifications = SpecDefinitionsFromPaths(specifications_paths.split(','), pseudo_clusters) + tests_finder = TestsFinder(configuration_directory, configuration_name) + + parser_config = TestParserConfig(pics, specifications, kwargs) + parser_builder_config = TestParserBuilderConfig(tests_finder.get(test_name), parser_config, hooks=TestParserLogger()) + parser_builder_config.options.stop_on_error = stop_on_error + while ctx: + ctx.obj = ParserGroup(parser_builder_config, pseudo_clusters) + ctx = ctx.parent + + +@runner_base.command() +@pass_parser_group +def parse(parser_group: ParserGroup): + """Parse the test suite.""" + runner_config = None + + runner = TestRunner() + return runner.run(parser_group.builder_config, runner_config) + + +@runner_base.command() +@pass_parser_group +def dry_run(parser_group: ParserGroup): + """Simulate a run of the test suite.""" + runner_config = TestRunnerConfig(hooks=TestRunnerLogger()) + + runner = TestRunner() + return runner.run(parser_group.builder_config, runner_config) + + +@runner_base.command() +@test_runner_options +@pass_parser_group +def run(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool): + """Run the test suite.""" + adapter = __import__(adapter, fromlist=[None]).Adapter(parser_group.builder_config.parser_config.definitions) + runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number) + runner_hooks = TestRunnerLogger(show_adapter_logs, show_adapter_logs_on_error) + runner_config = TestRunnerConfig(adapter, parser_group.pseudo_clusters, runner_options, runner_hooks) + + runner = TestRunner() + return runner.run(parser_group.builder_config, runner_config) + + +@runner_base.command() +@test_runner_options +@websocket_runner_options +@pass_parser_group +def websocket(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, server_address: str, server_port: int, server_path: str, server_name: str, server_arguments: str): + """Run the test suite using websockets.""" + adapter = __import__(adapter, fromlist=[None]).Adapter(parser_group.builder_config.parser_config.definitions) + runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number) + runner_hooks = TestRunnerLogger(show_adapter_logs, show_adapter_logs_on_error) + runner_config = TestRunnerConfig(adapter, parser_group.pseudo_clusters, runner_options, runner_hooks) + + if server_path is None and server_name: + paths_finder = PathsFinder() + server_path = paths_finder.get(server_name) + + websocket_runner_hooks = WebSocketRunnerLogger() + websocket_runner_config = WebSocketRunnerConfig( + server_address, server_port, server_path, server_arguments, websocket_runner_hooks) + + runner = WebSocketRunner(websocket_runner_config) + return runner.run(parser_group.builder_config, runner_config) + + +@runner_base.command() +@test_runner_options +@websocket_runner_options +@click.pass_context +def chiptool(ctx, *args, **kwargs): + """Run the test suite using chip-tool.""" + return ctx.forward(websocket) + + +@runner_base.command() +@test_runner_options +@websocket_runner_options +@click.pass_context +def app1(ctx, *args, **kwargs): + """Run the test suite using app1.""" + return ctx.forward(websocket) + + +@runner_base.command() +@test_runner_options +@websocket_runner_options +@click.pass_context +def app2(ctx, *args, **kwargs): + """Run the test suite using app2.""" + return ctx.forward(websocket) + + +if __name__ == '__main__': + success = False + try: + # By default click runs in standalone mode and it will handle exceptions and the + # different commands return values for us. For example it will set sys.exit to + # 0 if the test runs fails unless an exception is raised. Simple test failure + # does not raise exception but we want to set the exit code to 1. + # So standalone_mode is set to False to let us manage this exit behavior. + success = runner_base(standalone_mode=False) + except Exception: + print('') + traceback.print_exc() + + sys.exit(0 if success else 1) diff --git a/scripts/tests/yaml/tests_finder.py b/scripts/tests/yaml/tests_finder.py new file mode 100755 index 00000000000000..0eaa43fe5eaa05 --- /dev/null +++ b/scripts/tests/yaml/tests_finder.py @@ -0,0 +1,121 @@ +#!/usr/bin/env -S python3 -B +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os.path + +import click + +_JSON_FILE_EXTENSION = '.json' +_YAML_FILE_EXTENSION = '.yaml' +_KNOWN_PREFIX = 'Test_TC_' + +_KEYWORD_ALL_TESTS = 'all' +_DEFAULT_DIRECTORY = 'src/app/tests/suites/' + +_CI_CONFIGURATION_NAME = 'ciTests' +_MANUAL_CONFIGURATION_NAME = 'manualTests' + + +class TestsFinder: + def __init__(self, configuration_directory: str = _DEFAULT_DIRECTORY, configuration_name: str = _CI_CONFIGURATION_NAME): + self.__test_directory = _DEFAULT_DIRECTORY + self.__test_collections = self.__get_collections(configuration_directory, configuration_name) + + def get_default_configuration_directory() -> str: + return _DEFAULT_DIRECTORY + + def get_default_configuration_name() -> str: + return _CI_CONFIGURATION_NAME + + def get(self, test_name: str) -> list[str]: + test_names = [] + + if self.__test_collections and test_name == _KEYWORD_ALL_TESTS: + for collection_name in self.__test_collections.get('collection'): + for name in self.__test_collections.get(collection_name): + test_names.append(name) + elif self.__test_collections and self.__test_collections.get(test_name): + test_names = self.__test_collections.get(test_name) + else: + test_names.append(test_name) + + return self.__get_paths(test_names) + + def __get_collections(self, configuration_directory: str, configuration_name: str) -> list[str]: + if os.path.isfile(configuration_name): + configuration_filepath = configuration_name + elif os.path.isfile(os.path.join(configuration_directory, configuration_name + _JSON_FILE_EXTENSION)): + configuration_filepath = os.path.join(configuration_directory, configuration_name + _JSON_FILE_EXTENSION) + else: + configuration_filepath = None + + collections = None + if configuration_filepath: + with open(configuration_filepath) as file: + data = json.load(file) + if 'include' in data: + include_filepath = os.path.join(os.path.dirname(configuration_filepath), data.get('include')) + with open(include_filepath) as included_file: + collections = json.load(included_file) + else: + collections = data + + if collections and 'disable' in data: + disabled_tests = data.get('disable') + for disabled_test in disabled_tests: + for collection in collections: + if disabled_test in collections.get(collection): + collections.get(collection).remove(disabled_test) + + return collections + + def __get_paths(self, test_names: list[str]) -> list[str]: + paths = [] + + for name in test_names: + for root, dir, files in os.walk(self.__test_directory): + if name in files: + paths.append(os.path.join(root, name)) + elif (name + _YAML_FILE_EXTENSION) in files: + paths.append(os.path.join(root, name + _YAML_FILE_EXTENSION)) + elif (_KNOWN_PREFIX + name + _YAML_FILE_EXTENSION) in files: + paths.append(os.path.join(root, _KNOWN_PREFIX + name + _YAML_FILE_EXTENSION)) + + return paths + + +def test_finder_options(f): + f = click.option("--configuration_directory", type=click.Path(exists=True), required=True, show_default=True, + default=_DEFAULT_DIRECTORY, help='Path to the directory containing the tests configuration.')(f) + f = click.option("--configuration_name", type=str, required=True, show_default=True, + default=_CI_CONFIGURATION_NAME, help='Name of the collection configuration json file to use.')(f) + return f + + +@click.command() +@click.argument('test_name') +@test_finder_options +def run(test_name: str, configuration_directory: str, configuration_name: str): + """ Find a test or a set of tests.""" + tests_finder = TestsFinder(configuration_directory, configuration_name) + tests = tests_finder.get(test_name) + for test in tests: + print(test) + print(f'{len(tests)} tests found.') + + +if __name__ == '__main__': + run() diff --git a/scripts/tests/yaml/tests_logger.py b/scripts/tests/yaml/tests_logger.py new file mode 100755 index 00000000000000..4a0bd553a03047 --- /dev/null +++ b/scripts/tests/yaml/tests_logger.py @@ -0,0 +1,421 @@ +#!/usr/bin/env -S python3 -B +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import relative_importer # isort: split # noqa: F401 + +import json +import sys +import traceback +from dataclasses import dataclass + +import click +from matter_yamltests.errors import TestStepError, TestStepKeyError +from matter_yamltests.hooks import TestParserHooks, TestRunnerHooks, WebSocketRunnerHooks + + +def _strikethrough(str): + return '\u0336'.join(str[i:i+1] for i in range(0, len(str), 1)) + + +_SUCCESS = click.style(u'\N{check mark}', fg='green') +_FAILURE = click.style(u'\N{ballot x}', fg='red') +_WARNING = click.style(u'\N{warning sign}', fg='yellow') + + +class TestColoredLogPrinter(): + def __init__(self, log_format='[{module}] {message}'): + self.__log_format = log_format + self.__log_styles = { + 'Info': 'green', + 'Error': 'bright_red', + 'Debug': 'blue', + 'Others': 'white' + } + + def print(self, logs): + for log in logs: + fg = self.__log_styles[log.level] + click.secho(self.__log_format.format(module=log.module, message=log.message), fg=fg) + + +@dataclass +class ParserStrings: + start = 'Parsing {count} files.' + stop = '{state} Parsing finished in {duration}ms with {successes} success and {errors} errors.' + test_start = click.style('\t\tParsing: ', fg='white') + '{name}' + test_result = '\r{state} ' + click.style('{duration}ms', fg='white') + error_header = click.style('\t\tError at step {index}:', fg='white', bold=True) + error_line = click.style('\t\t{error_line}', fg='white') + + +class TestParserLogger(TestParserHooks): + def __init__(self): + self.__success = 0 + self.__errors = 0 + self.__strings = ParserStrings() + + def start(self, count: int): + print(self.__strings.start.format(count=count)) + + def stop(self, duration: int): + state = _FAILURE if self.__errors else _SUCCESS + success = click.style(self.__success, bold=True) + errors = click.style(self.__errors, bold=True) + print(self.__strings.stop.format(state=state, successes=success, errors=errors, duration=duration)) + + def test_start(self, name: str): + print(self.__strings.test_start.format(name=name), end='') + + def test_failure(self, exception: Exception, duration: int): + print(self.__strings.test_result.format(state=_FAILURE, duration=duration)) + + try: + raise exception + except TestStepError: + self.__print_step_exception(exception) + else: + traceback.print_exc() + + self.__errors += 1 + + def test_success(self, duration: int): + print(self.__strings.test_result.format(state=_SUCCESS, duration=duration)) + self.__success += 1 + + def __print_step_exception(self, exception: TestStepError): + if exception.context is None: + return + + print('') + print(self.__strings.error_header.format(index=exception.step_index)) + for line in exception.context.split('\n'): + if '__error_start__' in line and '__error_end__' in line: + before_end, after_end = line.split('__error_end__') + before_start, after_start = before_end.split('__error_start__') + line = click.style(before_start, fg='white') + line += click.style(after_start, fg='red', bold=True) + line += click.style(after_end, fg='white') + line += click.style(f' <-- {exception}', bold=True) + + print(self.__strings.error_line.format(error_line=line)) + + +@dataclass +class RunnerStrings: + start = '' + stop = 'Run finished in {duration}ms with {runned} steps runned and {skipped} steps skipped.' + test_start = 'Running: "{name}" with {count} steps.' + test_stop = '{state} Test finished in {duration}ms with {successes} success, {errors} errors and {warnings} warnings' + step_skipped = click.style('\t\t{index}. ' + _strikethrough('Running ') + '{name}', fg='white') + step_start = click.style('\t\t{index}. Running ', fg='white') + '{name}' + step_unknown = '' + step_result = '\r{state} ' + click.style('{duration}ms', fg='white') + result_entry = click.style('\t\t {state} [{category} check] {message}', fg='white') + result_log = '\t\t [{module}] {message}' + result_failure = '\t\t {key}: {value}' + error_header = click.style('\t\t Error at step {index}:', fg='white', bold=True) + error_line = click.style('\t\t {error_line}', fg='white') + + +class TestRunnerLogger(TestRunnerHooks): + def __init__(self, show_adapter_logs: bool = False, show_adapter_logs_on_error: bool = True): + self.__show_adapter_logs = show_adapter_logs + self.__show_adapter_logs_on_error = show_adapter_logs_on_error + self.__index = 1 + self.__successes = 0 + self.__warnings = 0 + self.__errors = 0 + self.__runned = 0 + self.__skipped = 0 + self.__strings = RunnerStrings() + self.__log_printer = TestColoredLogPrinter(self.__strings.result_log) + + def start(self, count: int): + print(self.__strings.start) + pass + + def stop(self, duration: int): + print(self.__strings.stop.format(runned=self.__runned, skipped=self.__skipped, duration=duration)) + + def test_start(self, name: str, count: int): + print(self.__strings.test_start.format(name=click.style(name, bold=True), count=click.style(count, bold=True))) + + def test_stop(self, duration: int): + if self.__errors: + state = _FAILURE + elif self.__warnings: + state = _WARNING + else: + state = _SUCCESS + + successes = click.style(self.__successes, bold=True) + errors = click.style(self.__errors, bold=True) + warnings = click.style(self.__warnings, bold=True) + print(self.__strings.test_stop.format(state=state, successes=successes, errors=errors, warnings=warnings, duration=duration)) + + def step_skipped(self, name: str): + print(self.__strings.step_skipped.format(index=self.__index, name=_strikethrough(name))) + + self.__index += 1 + self.__skipped += 1 + + def step_start(self, name: str): + print(self.__strings.step_start.format(index=self.__index, name=click.style(name, bold=True)), end='') + # flushing stdout such that the previous print statement is visible on the screen for long running tasks. + sys.stdout.flush() + + self.__index += 1 + + def step_unknown(self): + print(self.__strings.step_unknown) + + self.__runned += 1 + + def step_success(self, logger, logs, duration: int): + print(self.__strings.step_result.format(state=_SUCCESS, duration=duration)) + + self.__print_results(logger) + + if self.__show_adapter_logs: + self.__log_printer.print(logs) + + self.__successes += logger.successes + self.__warnings += logger.warnings + self.__errors += logger.errors + self.__runned += 1 + + def step_failure(self, logger, logs, duration: int, expected, received): + print(self.__strings.step_result.format(state=_FAILURE, duration=duration)) + + self.__print_results(logger) + + if self.__show_adapter_logs or self.__show_adapter_logs_on_error: + self.__log_printer.print(logs) + + has_failures_without_exception = False + for entry in logger.entries: + if entry.is_error() and entry.exception: + try: + raise entry.exception + except TestStepError as e: + self.__print_step_exception(e) + else: + traceback.print_exc() + elif entry.is_error(): + has_failures_without_exception = True + + if has_failures_without_exception: + self.__print_failure(expected, received) + + self.__successes += logger.successes + self.__warnings += logger.warnings + self.__errors += logger.errors + self.__runned += 1 + + def __print_step_exception(self, exception: TestStepError): + if exception.context is None: + return + + print('') + print(self.__strings.error_header.format(index=exception.step_index)) + for line in exception.context.split('\n'): + if '__error_start__' in line and '__error_end__' in line: + before_end, after_end = line.split('__error_end__') + before_start, after_start = before_end.split('__error_start__') + line = click.style(before_start, fg='white') + line += click.style(after_start, fg='red', bold=True) + line += click.style(after_end, fg='white') + line += click.style(f' <-- {exception}', bold=True) + + print(self.__strings.error_line.format(error_line=line)) + + def __print_results(self, logger): + for entry in logger.entries: + if entry.is_warning(): + state = _WARNING + elif entry.is_error(): + state = _FAILURE + else: + state = ' ' # Do not mark success to not make the output hard to read + + print(self.__strings.result_entry.format(state=state, category=entry.category, message=entry.message)) + + def __print_failure(self, expected_response, received_response): + expected_response = self.__prepare_data_for_printing(expected_response) + expected_value = json.dumps( + expected_response, sort_keys=True, indent=2, separators=(',', ': ')) + expected_value = expected_value.replace('\n', '\n\t\t ') + + received_response = self.__prepare_data_for_printing( + received_response) + received_value = json.dumps( + received_response, sort_keys=True, indent=2, separators=(',', ': ')) + received_value = received_value.replace('\n', '\n\t\t ') + print(self.__strings.result_failure.format(key=click.style("Expected", bold=True), value=expected_value)) + print(self.__strings.result_failure.format(key=click.style("Received", bold=True), value=received_value)) + + def __prepare_data_for_printing(self, data): + if isinstance(data, bytes): + return data.decode('unicode_escape') + elif isinstance(data, list): + return [self.__prepare_data_for_printing(entry) for entry in data] + elif isinstance(data, dict): + result = {} + for key, value in data.items(): + result[key] = self.__prepare_data_for_printing(value) + return result + + return data + + +@dataclass +class WebSocketRunnerStrings: + connecting = click.style('\t\tConnecting: ' + click.style('{url}', bold=True), fg='white') + abort = 'Connecting to {url} failed.' + success = click.style(f'\r{_SUCCESS} {{duration}}ms', fg='white') + failure = click.style(f'\r{_WARNING} {{duration}}ms', fg='white') + retry = click.style('\t\t Retrying in {interval} seconds.', fg='white') + + +class WebSocketRunnerLogger(WebSocketRunnerHooks): + def __init__(self): + self.__strings = WebSocketRunnerStrings() + + def connecting(self, url: str): + print(self.__strings.connecting.format(url=url), end='') + sys.stdout.flush() + + def abort(self, url: str): + print(self.__strings.abort.format(url=url)) + + def success(self, duration: int): + print(self.__strings.success.format(duration=duration)) + + def failure(self, duration: int): + print(self.__strings.failure.format(duration=duration)) + + def retry(self, interval_between_retries_in_seconds: int): + print(self.__strings.retry.format(interval=interval_between_retries_in_seconds)) + + +# +# Everything below this comment is for testing purposes only. +# It is here to quickly check the look and feel of the output +# that is produced via the different hooks. +# +@click.group() +def simulate(): + pass + + +@simulate.command() +def parser(): + """Simulate parsing tests.""" + parser_logger = TestParserLogger() + + test_step = { + 'label': 'This is a fake test step', + 'nodeId': 1, + 'cluster': 'TestStepCluster', + 'commandd': 'WrongCommandKey', + 'attribute': 'TestStepAttribute', + } + + test_step + + parser_logger.start(99) + parser_logger.test_start('test.yaml') + parser_logger.test_success(10) + parser_logger.test_start('test2.yaml') + exception = TestStepKeyError(test_step, 'commandd') + exception.update_context(test_step, 1) + parser_logger.test_failure(exception, 200) + parser_logger.stop(10 + 200) + + +@simulate.command() +def runner(): + """Simulate running tests.""" + runner_logger = TestRunnerLogger() + + class TestLogger: + def __init__(self, entries=[], successes=0, warnings=0, errors=0): + self.entries = entries + self.successes = successes + self.warnings = warnings + self.errors = errors + + class LogEntry: + def __init__(self, message, module='CTL', level='Others'): + self.message = message + self.module = module + self.level = level + + success_logger = TestLogger(successes=3) + error_logger = TestLogger(errors=2) + + expected_response = {} + received_response = {'error': 'UNSUPPORTED_COMMAND'} + + empty_logs = [] + other_logs = [ + LogEntry('This is a message without a category'), + LogEntry('This is an info message', level='Info'), + LogEntry('This is an error message', level='Error'), + LogEntry('This is a debug message', level='Debug'), + ] + + runner_logger.start(99) + runner_logger.test_start('test.yaml', 23) + runner_logger.step_start('First Step') + runner_logger.step_success(success_logger, empty_logs, 1234) + runner_logger.step_start('Second Step') + runner_logger.step_failure(error_logger, other_logs, 4321, expected_response, received_response) + runner_logger.step_skipped('Third Step') + runner_logger.step_start('Fourth Step') + runner_logger.step_unknown() + runner_logger.test_stop(1234 + 4321) + runner_logger.stop(123456) + + +@simulate.command() +def websocket_abort(): + """Simulate a websocket connection aborting.""" + websocket_runner_logger = WebSocketRunnerLogger() + url = 'ws://localhost:9002' + + websocket_runner_logger.connecting(url) + websocket_runner_logger.failure(30) + websocket_runner_logger.retry(1) + websocket_runner_logger.connecting(url) + websocket_runner_logger.failure(30) + websocket_runner_logger.abort(url) + + +@simulate.command() +def websocket_connected(): + """Simulate a websocket connection that successfuly connects.""" + websocket_runner_logger = WebSocketRunnerLogger() + url = 'ws://localhost:9002' + + websocket_runner_logger.connecting(url) + websocket_runner_logger.failure(30) + websocket_runner_logger.retry(1) + websocket_runner_logger.connecting(url) + websocket_runner_logger.success(30) + + +if __name__ == '__main__': + simulate()