From 800cedb75ebde269ebc128394d5ac8055e3126e6 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Tue, 29 Aug 2023 20:45:34 +0200 Subject: [PATCH] [Tizen] Run matter unit tests on QEMU (#26563) * Run unit tests on Tizen platform under QEMU * Run all generated unit test executables * Move failing tests to XFAIL group * Manual for debugging tests on QEMU * Add "Run test (Linux x64)" to VSCode debugger * Print failed tests * Drop XFAIL since now all tests pass on Tizen * Do not duplicate example builds with QEMU tests * Do not add rootshell to the word list * Revert defines reordering * Add TODO for GN test assets --------- Co-authored-by: Andrei Litvin --- .github/workflows/examples-tizen.yaml | 4 +- .vscode/launch.json | 19 ++++++- build/chip/chip_test.gni | 32 +++++------ build/chip/chip_test_group.gni | 12 +++-- build/chip/chip_test_suite.gni | 10 ++-- build/chip/tests.gni | 13 +++-- scripts/build/builders/tizen.py | 3 ++ src/test_driver/tizen/BUILD.gn | 5 +- src/test_driver/tizen/README.md | 17 ++++++ src/test_driver/tizen/chip_tests/BUILD.gn | 49 +++++++++++++++++ src/test_driver/tizen/chip_tests/runner.sh | 53 +++++++++++++++++++ .../integration_tests/lighting-app/BUILD.gn | 13 +++-- third_party/tizen/tizen_sdk.gni | 3 ++ 13 files changed, 197 insertions(+), 36 deletions(-) create mode 100644 src/test_driver/tizen/chip_tests/BUILD.gn create mode 100755 src/test_driver/tizen/chip_tests/runner.sh diff --git a/.github/workflows/examples-tizen.yaml b/.github/workflows/examples-tizen.yaml index be99cb029bb293..b70990ccc9c2c1 100644 --- a/.github/workflows/examples-tizen.yaml +++ b/.github/workflows/examples-tizen.yaml @@ -25,7 +25,7 @@ concurrency: env: CHIP_NO_LOG_TIMESTAMPS: true - + jobs: tizen: name: Tizen @@ -54,10 +54,8 @@ jobs: "./scripts/build/build_examples.py \ --enable-flashbundle \ --target tizen-arm-all-clusters \ - --target tizen-arm-all-clusters-minimal-no-wifi \ --target tizen-arm-chip-tool-ubsan \ --target tizen-arm-light \ - --target tizen-arm-light-no-ble-no-wifi \ build \ --copy-artifacts-to out/artifacts \ " diff --git a/.vscode/launch.json b/.vscode/launch.json index ed80ae59c95e23..0ed822ad0251a9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,12 +20,20 @@ }, { "name": "Run application (Linux x64)", - "type": "lldb", + "type": "cppdbg", "request": "launch", "program": "${workspaceFolder}/out/${input:outAppLinux}", "args": [], "cwd": "${workspaceFolder}" }, + { + "name": "Run test (Linux x64)", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/out/${input:outTestLinux}", + "args": [], + "cwd": "${workspaceFolder}" + }, { "name": "QRCode Tests", "type": "cppdbg", @@ -486,6 +494,15 @@ "description": "Select the application to run" } }, + { + "type": "command", + "id": "outTestLinux", + "command": "shellCommand.execute", + "args": { + "command": "find ${workspaceFolder}/out/linux-x64-*/tests -type f -executable |sort |sed 's$${workspaceFolder}/out/$$'", + "description": "Select the test to run" + } + }, { "type": "pickString", "id": "mbedDebugProfile", diff --git a/build/chip/chip_test.gni b/build/chip/chip_test.gni index 08fbb3e01a429c..8c0d498bb68074 100644 --- a/build/chip/chip_test.gni +++ b/build/chip/chip_test.gni @@ -41,21 +41,23 @@ if (chip_link_tests) { group(_test_name + "_lib") { } - pw_python_action(_test_name + "_run") { - deps = [ ":${_test_name}" ] - inputs = [ pw_unit_test_AUTOMATIC_RUNNER ] - module = "pw_unit_test.test_runner" - python_deps = [ - "$dir_pw_cli/py", - "$dir_pw_unit_test/py", - ] - args = [ - "--runner", - rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir), - "--test", - rebase_path("$_test_output_dir/$_test_name", root_build_dir), - ] - stamp = true + if (chip_pw_run_tests) { + pw_python_action(_test_name + "_run") { + deps = [ ":${_test_name}" ] + inputs = [ pw_unit_test_AUTOMATIC_RUNNER ] + module = "pw_unit_test.test_runner" + python_deps = [ + "$dir_pw_cli/py", + "$dir_pw_unit_test/py", + ] + args = [ + "--runner", + rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir), + "--test", + rebase_path("$_test_output_dir/$_test_name", root_build_dir), + ] + stamp = true + } } } } else { diff --git a/build/chip/chip_test_group.gni b/build/chip/chip_test_group.gni index bb4c9def4bfb9b..aa0bc841ec5ac1 100644 --- a/build/chip/chip_test_group.gni +++ b/build/chip/chip_test_group.gni @@ -60,11 +60,13 @@ template("chip_test_group") { } } - group("${_test_group_name}_run") { - if (chip_link_tests) { - deps = [] - foreach(_test, invoker.deps) { - deps += [ get_label_info(_test, "label_no_toolchain") + "_run" ] + if (chip_pw_run_tests) { + group("${_test_group_name}_run") { + if (chip_link_tests) { + deps = [] + foreach(_test, invoker.deps) { + deps += [ get_label_info(_test, "label_no_toolchain") + "_run" ] + } } } } diff --git a/build/chip/chip_test_suite.gni b/build/chip/chip_test_suite.gni index 336450ca459487..1640988016c25f 100644 --- a/build/chip/chip_test_suite.gni +++ b/build/chip/chip_test_suite.gni @@ -151,10 +151,12 @@ template("chip_test_suite") { } } - group("${_suite_name}_run") { - deps = [] - foreach(_test, tests) { - deps += [ ":${_test}_run" ] + if (chip_pw_run_tests) { + group("${_suite_name}_run") { + deps = [] + foreach(_test, tests) { + deps += [ ":${_test}_run" ] + } } } } else { diff --git a/build/chip/tests.gni b/build/chip/tests.gni index 70218c49728ed8..61a2ad3e685682 100644 --- a/build/chip/tests.gni +++ b/build/chip/tests.gni @@ -24,10 +24,17 @@ declare_args() { declare_args() { # Build executables for running individual tests. - chip_link_tests = - chip_build_tests && (current_os == "linux" || current_os == "mac") && - current_cpu == target_cpu + chip_link_tests = chip_build_tests && + (current_os == "linux" || current_os == "mac" || + current_os == "tizen") && current_cpu == target_cpu +} +declare_args() { + # Run tests with pigweed test runner. + chip_pw_run_tests = chip_link_tests && current_os != "tizen" +} + +declare_args() { # Use source_set instead of static_lib for tests. chip_build_test_static_libraries = chip_device_platform != "efr32" } diff --git a/scripts/build/builders/tizen.py b/scripts/build/builders/tizen.py index e62e8fdffd11e9..966b5da62207de 100644 --- a/scripts/build/builders/tizen.py +++ b/scripts/build/builders/tizen.py @@ -113,6 +113,9 @@ def __init__(self, except FileNotFoundError: pass + if app == TizenApp.TESTS: + self.extra_gn_options.append('chip_build_tests=true') + if not enable_ble: self.extra_gn_options.append('chip_config_network_layer_ble=false') if not enable_thread: diff --git a/src/test_driver/tizen/BUILD.gn b/src/test_driver/tizen/BUILD.gn index 9d7c863d3ce7ca..5374fa75ed82a6 100644 --- a/src/test_driver/tizen/BUILD.gn +++ b/src/test_driver/tizen/BUILD.gn @@ -14,5 +14,8 @@ group("check") { testonly = true - deps = [ "integration_tests/lighting-app:check" ] + deps = [ + "chip_tests:check", + "integration_tests/lighting-app:check", + ] } diff --git a/src/test_driver/tizen/README.md b/src/test_driver/tizen/README.md index ce36f327d7abb1..eb7ef0cb71683f 100644 --- a/src/test_driver/tizen/README.md +++ b/src/test_driver/tizen/README.md @@ -55,3 +55,20 @@ gn gen --check --fail-on-unused-args \ # Run Tizen QEMU-based tests ninja -C out/tizen-check check ``` + +## Debugging Tests on QEMU + +When running tests with `ninja -C out/tizen-check check`, the test driver prints +the command to run the test on QEMU. Please search for the following line in the +output of the `ninja` command: `INFO:root:run: qemu-system-arm`. Make sure that +your terminal does not limit the number of lines printed, or simply redirect the +output to a file as follows: + +```sh +ninja -C out/tizen-check check > ninja-tizen-check.log 2>&1 +``` + +Then, use the run command and add the `rootshell` keyword to kernel arguments +passed to QEMU (the string after the `-append` option). This will run QEMU, but +instead of running the test, it will drop you to the shell. From there, you can +run the test manually by typing `/mnt/chip/runner.sh`. diff --git a/src/test_driver/tizen/chip_tests/BUILD.gn b/src/test_driver/tizen/chip_tests/BUILD.gn new file mode 100644 index 00000000000000..829371df7cc620 --- /dev/null +++ b/src/test_driver/tizen/chip_tests/BUILD.gn @@ -0,0 +1,49 @@ +# Copyright (c) 2020 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build_overrides/chip.gni") +import("//build_overrides/tizen.gni") + +import("${tizen_sdk_build_root}/tizen_sdk.gni") + +tizen_qemu_mkisofs("chip-tests-runner") { + runner = "runner.sh" + + # Build CHIP unit tests. + deps = [ "${chip_root}/src:tests" ] + + # Bundle all created unit tests. + # TODO (arkq): Collect all unit tests in a way that we could add all targets + # to the assets list. Right now QEMU runner dependencies are not + # tracked properly. Changes to the unit tests will not trigger + # rebuild of the ISO image, so the test will be run with old + # binaries. + assets_non_tracked = [ rebase_path("${root_build_dir}/tests") ] +} + +tizen_qemu_run("chip-tests") { + # Enable network support, so Tizen can obtain current date/time from the + # network. Correct date/time is required for the commissioning process - + # attestation will fail otherwise. + virtio_net = true + + deps = [ ":chip-tests-runner" ] + mkisofs_outputs = get_target_outputs(":chip-tests-runner") + iso_image = rebase_path(mkisofs_outputs[0]) +} + +group("check") { + deps = [ ":chip-tests" ] + testonly = true +} diff --git a/src/test_driver/tizen/chip_tests/runner.sh b/src/test_driver/tizen/chip_tests/runner.sh new file mode 100755 index 00000000000000..1cc401626cb230 --- /dev/null +++ b/src/test_driver/tizen/chip_tests/runner.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Print CHIP logs on stdout +dlogutil CHIP & + +FAILED=() +STATUS=0 + +# Run all executables in the /mnt/chip directory except the runner.sh script +while IFS= read -r TEST; do + + NAME=$(basename "$TEST") + + echo + echo "RUN: $NAME" + + RV=0 + "$TEST" || RV=$? + + if [ "$RV" -eq 0 ]; then + echo -e "DONE: \e[32mSUCCESS\e[0m" + else + FAILED+=("$NAME") + STATUS=$((STATUS + 1)) + echo -e "DONE: \e[31mFAIL\e[0m" + fi + +done < <(find /mnt/chip -type f -executable ! -name runner.sh) + +if [ ! "$STATUS" -eq 0 ]; then + echo + echo "### FAILED: ${FAILED[*]}" +fi + +exit "$STATUS" diff --git a/src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn b/src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn index 612d1cfb0c7728..15c16d058af5b5 100644 --- a/src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn +++ b/src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn @@ -17,7 +17,7 @@ import("//build_overrides/tizen.gni") import("${tizen_sdk_build_root}/tizen_sdk.gni") -tizen_qemu_mkisofs("test-runner") { +tizen_qemu_mkisofs("lighting-app-runner") { runner = "runner.sh" # Build applications used in the test. @@ -34,13 +34,18 @@ tizen_qemu_mkisofs("test-runner") { ] } -tizen_qemu_run("check") { +tizen_qemu_run("lighting-app") { # Enable network support, so Tizen can obtain current date/time from the # network. Correct date/time is required for the commissioning process - # attestation will fail otherwise. virtio_net = true - deps = [ ":test-runner" ] - mkisofs_outputs = get_target_outputs(":test-runner") + deps = [ ":lighting-app-runner" ] + mkisofs_outputs = get_target_outputs(":lighting-app-runner") iso_image = rebase_path(mkisofs_outputs[0]) } + +group("check") { + deps = [ ":lighting-app" ] + testonly = true +} diff --git a/third_party/tizen/tizen_sdk.gni b/third_party/tizen/tizen_sdk.gni index 70242ee1c3dbf6..3c8704a8491a13 100644 --- a/third_party/tizen/tizen_sdk.gni +++ b/third_party/tizen/tizen_sdk.gni @@ -198,6 +198,9 @@ template("tizen_qemu_mkisofs") { args += invoker.assets inputs += invoker.assets } + if (defined(invoker.assets_non_tracked)) { + args += invoker.assets_non_tracked + } script = "${build_root}/gn_run_binary.py" }