From 0ff49d6f7f954bebc86509ea0a0f8f8e9d096357 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 24 Jul 2023 17:16:24 -0400 Subject: [PATCH] Add support for `--trace-to` for python and use it in repl tests (#28179) * Start adding tracing start/stop functions * Add raii-like support for tracing * Add raii-like support for tracing * Fix compile logic * Switch linux, mac, android to C++17 by default * Start outputting trace data * Upload traces that are gathered * Fix names * Allow placeholders in script args too * Allow from-string tracing * Do not newline-separate restyle path otherwise only the first argument is processed * Restyle * Add some additional types * Minor python fixes * Import ctypes * Things run now * Add trace bits to our tests * Undo restyle-diff change * Fix some typos in naming * Add perfetto for darwin too * mobile-device-test.py does not suppor trace-to yet * Make mobile device test also be able to trace. Mobile device test seems super slow * Restyled by autopep8 * Restyled by isort --------- Co-authored-by: Andrei Litvin Co-authored-by: Restyled.io --- .github/workflows/tests.yaml | 44 +++--- .gitmodules | 2 +- scripts/tests/run_python_test.py | 8 +- src/controller/python/BUILD.gn | 11 ++ src/controller/python/chip/native/__init__.py | 8 +- .../python/chip/tracing/TracingSetup.cpp | 104 +++++++++++++++ .../python/chip/tracing/__init__.py | 125 ++++++++++++++++++ .../test/test_scripts/mobile-device-test.py | 15 ++- src/python_testing/matter_testing_support.py | 70 +++++----- 9 files changed, 331 insertions(+), 56 deletions(-) create mode 100644 src/controller/python/chip/tracing/TracingSetup.cpp create mode 100644 src/controller/python/chip/tracing/__init__.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 99f7055113d2ec..c462203ae6cd0d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -437,24 +437,25 @@ jobs: " - name: Run Tests run: | - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_DeviceBasicComposition.py" --script-args "--storage-path admin_storage.json --manual-code 10054912339"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_SC_3_6.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_DA_1_7.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log --enable-key 000102030405060708090a0b0c0d0e0f" --script "src/python_testing/TC_TestEventTrigger.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_ACE_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_ACE_1_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_ACE_1_3.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_CGEN_2_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_DA_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_TIMESYNC_3_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_DA_1_5.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_IDM_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_FAN_3_3.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_FAN_3_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:log" --script "src/python_testing/TC_FAN_3_5.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"' - scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TestMatterTestingSupport.py"' + mkdir -p out/trace_data + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_DeviceBasicComposition.py" --script-args "--storage-path admin_storage.json --manual-code 10054912339 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_SC_3_6.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_DA_1_7.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json --enable-key 000102030405060708090a0b0c0d0e0f" --script "src/python_testing/TC_TestEventTrigger.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_ACE_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_ACE_1_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --int-arg PIXIT.ACE.APPENDPOINT:1 PIXIT.ACE.APPDEVTYPEID:0x0100 --string-arg PIXIT.ACE.APPCLUSTER:OnOff PIXIT.ACE.APPATTRIBUTE:OnOff --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_ACE_1_3.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_CGEN_2_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_DA_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_TIMESYNC_3_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_DA_1_5.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_IDM_1_2.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_FAN_3_3.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_FAN_3_4.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace-to json:out/trace_data/app-{SCRIPT_BASE_NAME}.json" --script "src/python_testing/TC_FAN_3_5.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --script "src/python_testing/TestMatterTestingSupport.py" --script-args "--trace-to json:out/trace_data/test-{SCRIPT_BASE_NAME}.json --trace-to perfetto:out/trace_data/test-{SCRIPT_BASE_NAME}.perfetto"' - name: Uploading core files uses: actions/upload-artifact@v3 if: ${{ failure() && !env.ACT }} @@ -524,6 +525,13 @@ jobs: path: /cores/ # Cores are big; don't hold on to them too long. retention-days: 5 + - name: Uploading traces on failure + uses: actions/upload-artifact@v3 + if: ${{ failure() && !env.ACT }} + with: + name: trace-data-python-repl + path: out/trace_data/ + retention-days: 5 - name: Uploading diagnostic logs uses: actions/upload-artifact@v3 if: ${{ failure() && !env.ACT }} diff --git a/.gitmodules b/.gitmodules index 990cb20f276572..15082161f151d6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -307,7 +307,7 @@ path = third_party/perfetto/repo url = https://github.com/google/perfetto.git branch = master - platforms = linux,android + platforms = linux,android,darwin [submodule "third_party/asr/components"] path = third_party/asr/components url = https://github.com/asriot/asriot_components.git diff --git a/scripts/tests/run_python_test.py b/scripts/tests/run_python_test.py index c607a5d07cf40f..33931accb909f9 100755 --- a/scripts/tests/run_python_test.py +++ b/scripts/tests/run_python_test.py @@ -17,6 +17,7 @@ import datetime import logging import os +import os.path import queue import re import shlex @@ -72,7 +73,7 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st @click.option("--factoryreset", is_flag=True, help='Remove app config and repl configs (/tmp/chip* and /tmp/repl*) before running the tests.') @click.option("--app-args", type=str, default='', - help='The extra arguments passed to the device.') + help='The extra arguments passed to the device. Can use placholders like {SCRIPT_BASE_NAME}') @click.option("--script", type=click.Path(exists=True), default=os.path.join(DEFAULT_CHIP_ROOT, 'src', 'controller', @@ -81,10 +82,13 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st 'test_scripts', 'mobile-device-test.py'), help='Test script to use.') @click.option("--script-args", type=str, default='', - help='Path to the test script to use, omit to use the default test script (mobile-device-test.py).') + help='Script arguments, can use placeholders like {SCRIPT_BASE_NAME}.') @click.option("--script-gdb", is_flag=True, help='Run script through gdb') def main(app: str, factoryreset: bool, app_args: str, script: str, script_args: str, script_gdb: bool): + app_args = app_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0]) + script_args = script_args.replace('{SCRIPT_BASE_NAME}', os.path.splitext(os.path.basename(script))[0]) + if factoryreset: # Remove native app config retcode = subprocess.call("rm -rf /tmp/chip* /tmp/repl*", shell=True) diff --git a/src/controller/python/BUILD.gn b/src/controller/python/BUILD.gn index c76c64c65d7df4..375a65cf7becbc 100644 --- a/src/controller/python/BUILD.gn +++ b/src/controller/python/BUILD.gn @@ -78,6 +78,7 @@ shared_library("ChipDeviceCtrl") { "chip/internal/CommissionerImpl.cpp", "chip/logging/LoggingRedirect.cpp", "chip/native/PyChipError.cpp", + "chip/tracing/TracingSetup.cpp", "chip/utils/DeviceProxyUtils.cpp", ] defines += [ "CHIP_CONFIG_MAX_GROUPS_PER_FABRIC=50" ] @@ -120,8 +121,16 @@ shared_library("ChipDeviceCtrl") { public_deps += [ "${chip_root}/src/controller/data_model", "${chip_root}/src/credentials:file_attestation_trust_store", + "${chip_root}/src/tracing/json", + "${chip_root}/src/tracing/perfetto", + "${chip_root}/src/tracing/perfetto:file_output", "${chip_root}/third_party/jsoncpp", ] + + deps = [ + "${chip_root}/src/tracing/perfetto:event_storage", + "${chip_root}/src/tracing/perfetto:simple_initialization", + ] } else { public_deps += [ "$chip_data_model" ] } @@ -238,6 +247,7 @@ chip_python_wheel_action("chip-core") { "chip/setup_payload/__init__.py", "chip/setup_payload/setup_payload.py", "chip/storage/__init__.py", + "chip/tracing/__init__.py", "chip/utils/CommissioningBuildingBlocks.py", "chip/utils/__init__.py", "chip/yaml/__init__.py", @@ -292,6 +302,7 @@ chip_python_wheel_action("chip-core") { "chip.clusters", "chip.setup_payload", "chip.storage", + "chip.tracing", ] if (!chip_controller) { diff --git a/src/controller/python/chip/native/__init__.py b/src/controller/python/chip/native/__init__.py index 26e394e61fca6a..7801129c948532 100644 --- a/src/controller/python/chip/native/__init__.py +++ b/src/controller/python/chip/native/__init__.py @@ -197,5 +197,9 @@ def Init(bluetoothAdapter: int = None): _GetLibraryHandle(False).pychip_CommonStackInit(ctypes.c_char_p(params)) -def GetLibraryHandle(): - return _GetLibraryHandle(True) +class HandleFlags(enum.Flag): + REQUIRE_INITIALIZATION = enum.auto() + + +def GetLibraryHandle(flags=HandleFlags.REQUIRE_INITIALIZATION): + return _GetLibraryHandle(HandleFlags.REQUIRE_INITIALIZATION in flags) diff --git a/src/controller/python/chip/tracing/TracingSetup.cpp b/src/controller/python/chip/tracing/TracingSetup.cpp new file mode 100644 index 00000000000000..d09a655b74c633 --- /dev/null +++ b/src/controller/python/chip/tracing/TracingSetup.cpp @@ -0,0 +1,104 @@ +/* + * + * Copyright (c) 2023 Project CHIP Authors + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +using chip::DeviceLayer::PlatformMgr; + +class ScopedStackLock +{ +public: + ScopedStackLock() { PlatformMgr().LockChipStack(); } + + ~ScopedStackLock() { PlatformMgr().UnlockChipStack(); } +}; + +chip::Tracing::Json::JsonBackend gJsonBackend; + +chip::Tracing::Perfetto::FileTraceOutput gPerfettoFileOutput; +chip::Tracing::Perfetto::PerfettoBackend gPerfettoBackend; + +} // namespace + +extern "C" void pychip_tracing_start_json_log(const char * file_name) +{ + + ScopedStackLock lock; + + gJsonBackend.CloseFile(); // just in case, ensure no file output + chip::Tracing::Register(gJsonBackend); +} + +extern "C" PyChipError pychip_tracing_start_json_file(const char * file_name) +{ + ScopedStackLock lock; + + CHIP_ERROR err = gJsonBackend.OpenFile(file_name); + if (err != CHIP_NO_ERROR) + { + return ToPyChipError(err); + } + chip::Tracing::Register(gJsonBackend); + return ToPyChipError(CHIP_NO_ERROR); +} + +extern "C" void pychip_tracing_start_perfetto_system() +{ + ScopedStackLock lock; + + chip::Tracing::Perfetto::Initialize(perfetto::kSystemBackend); + chip::Tracing::Perfetto::RegisterEventTrackingStorage(); + chip::Tracing::Register(gPerfettoBackend); +} + +extern "C" PyChipError pychip_tracing_start_perfetto_file(const char * file_name) +{ + ScopedStackLock lock; + + chip::Tracing::Perfetto::Initialize(perfetto::kInProcessBackend); + chip::Tracing::Perfetto::RegisterEventTrackingStorage(); + + CHIP_ERROR err = gPerfettoFileOutput.Open(file_name); + if (err != CHIP_NO_ERROR) + { + return ToPyChipError(err); + } + chip::Tracing::Register(gPerfettoBackend); + + return ToPyChipError(CHIP_NO_ERROR); +} + +extern "C" void pychip_tracing_stop() +{ + ScopedStackLock lock; + + chip::Tracing::Perfetto::FlushEventTrackingStorage(); + gPerfettoFileOutput.Close(); + chip::Tracing::Unregister(gPerfettoBackend); + chip::Tracing::Unregister(gJsonBackend); +} diff --git a/src/controller/python/chip/tracing/__init__.py b/src/controller/python/chip/tracing/__init__.py new file mode 100644 index 00000000000000..fc6ee35f3b1bb9 --- /dev/null +++ b/src/controller/python/chip/tracing/__init__.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +from enum import Enum, auto +from typing import Optional + +import chip.native +from chip.native import PyChipError + + +def _GetTracingLibraryHandle() -> ctypes.CDLL: + """ Get the native library handle with tracing methods initialized. + + Retreives the CHIP native library handle and attaches signatures to + native methods. + """ + + # Getting a handle without requiring init, as tracing methods + # do not require chip stack startup + handle = chip.native.GetLibraryHandle(chip.native.HandleFlags(0)) + + # Uses one of the type decorators as an indicator for everything being + # initialized. + if not handle.pychip_tracing_start_json_file.argtypes: + setter = chip.native.NativeLibraryHandleMethodArguments(handle) + + setter.Set('pychip_tracing_start_json_log', None, []) + setter.Set('pychip_tracing_start_json_file', PyChipError, [ctypes.c_char_p]) + + setter.Set('pychip_tracing_start_perfetto_system', None, []) + setter.Set('pychip_tracing_start_perfetto_file', PyChipError, [ctypes.c_char_p]) + + setter.Set('pychip_tracing_stop', None, []) + + return handle + + +class TraceType(Enum): + JSON = auto() + PERFETTO = auto() + + +def StartTracingTo(trace_type: TraceType, file_name: Optional[str] = None): + """ + Initiate tracing to the specified destination. + + Note that only one active trace can exist of a given type (i.e. cannot trace both + to files and logs/system). + """ + handle = _GetTracingLibraryHandle() + + if trace_type == TraceType.JSON: + if file_name is None: + handle.pychip_tracing_start_json_log() + else: + handle.pychip_tracing_start_json_file(file_name.encode('utf-8')).raise_on_error() + elif trace_type == TraceType.PERFETTO: + if file_name is None: + handle.pychip_tracing_start_perfetto_system() + else: + handle.pychip_tracing_start_perfetto_file(file_name.encode('utf-8')).raise_on_error() + else: + raise ValueError("unknown trace type") + + +def StopTracing(): + """ + Make sure tracing is stopped. + + MUST be called before application exits. + """ + _GetTracingLibraryHandle().pychip_tracing_stop() + + +class TracingContext: + """Allows scoped enter/exit for tracing, like: + + with TracingContext() as tracing: + tracing.Start(TraceType.JSON) + # ... + + """ + + def Start(self, trace_type: TraceType, file_name: Optional[str] = None): + StartTracingTo(trace_type, file_name) + + def StartFromString(self, destination: str): + """ + Convert a human string to a perfetto start. + + Supports json:log, json:path, perfetto, perfetto:path + """ + if destination == 'perfetto': + self.Start(TraceType.PERFETTO) + elif destination == 'json:log': + self.Start(TraceType.JSON) + elif destination.startswith("json:"): + self.Start(TraceType.JSON, destination[5:]) + elif destination.startswith("perfetto:"): + self.Start(TraceType.PERFETTO, destination[9:]) + else: + raise ValueError("Invalid trace-to destination: %r", destination) + + def __init__(self): + pass + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + StopTracing() diff --git a/src/controller/python/test/test_scripts/mobile-device-test.py b/src/controller/python/test/test_scripts/mobile-device-test.py index 09f13ccb444843..87e0a9acb6569b 100755 --- a/src/controller/python/test/test_scripts/mobile-device-test.py +++ b/src/controller/python/test/test_scripts/mobile-device-test.py @@ -27,6 +27,7 @@ import click import coloredlogs from base import BaseTestHelper, FailIfNot, SetTestSet, TestFail, TestTimeout, logger +from chip.tracing import TracingContext from cluster_objects import ClusterObjectTests from network_commissioning import NetworkCommissioningTests @@ -246,8 +247,12 @@ def do_tests(controller_nodeid, device_nodeid, address, timeout, discriminator, default='', type=str, help="Path that contains valid and trusted PAA Root Certificates.") +@click.option('--trace-to', + multiple=True, + default=[], + help="Trace location") def run(controller_nodeid, device_nodeid, address, timeout, discriminator, setup_pin, enable_test, disable_test, log_level, - log_format, print_test_list, paa_trust_store_path): + log_format, print_test_list, paa_trust_store_path, trace_to): coloredlogs.install(level=log_level, fmt=log_format, logger=logger) if print_test_list: @@ -267,8 +272,12 @@ def run(controller_nodeid, device_nodeid, address, timeout, discriminator, setup logger.info(f"\tEnabled Tests: {enable_test}") logger.info(f"\tDisabled Tests: {disable_test}") SetTestSet(enable_test, disable_test) - do_tests(controller_nodeid, device_nodeid, address, timeout, - discriminator, setup_pin, paa_trust_store_path) + with TracingContext() as tracing_ctx: + for destination in trace_to: + tracing_ctx.StartFromString(destination) + + do_tests(controller_nodeid, device_nodeid, address, timeout, + discriminator, setup_pin, paa_trust_store_path) if __name__ == "__main__": diff --git a/src/python_testing/matter_testing_support.py b/src/python_testing/matter_testing_support.py index b51da99aed15e6..7570f3ddcbe415 100644 --- a/src/python_testing/matter_testing_support.py +++ b/src/python_testing/matter_testing_support.py @@ -51,6 +51,7 @@ from chip.clusters.Attribute import EventReadResult, SubscriptionTransaction from chip.interaction_model import InteractionModelError, Status from chip.storage import PersistentStorage +from chip.tracing import TracingContext from mobly import asserts, base_test, signals, utils from mobly.config_parser import ENV_MOBLY_LOGPATH, TestRunConfig from mobly.test_runner import TestRunner @@ -265,6 +266,8 @@ class MatterTestConfig: # If this is set, we will reuse root of trust keys at that location chip_tool_credentials_path: Optional[pathlib.Path] = None + trace_to: List[str] = field(default_factory=list) + class ClusterMapper: """Describe clusters/attributes using schema names.""" @@ -817,6 +820,7 @@ def convert_args_to_matter_config(args: argparse.Namespace) -> MatterTestConfig: config.pics = {} if args.PICS is None else read_pics_from_file(args.PICS) config.controller_node_id = args.controller_node_id + config.trace_to = args.trace_to # Accumulate all command-line-passed named args all_global_args = [] @@ -847,7 +851,8 @@ def parse_matter_test_args(argv: List[str]) -> MatterTestConfig: type=str, metavar='test_a test_b...', help='A list of tests in the test class to execute.') - + basic_group.add_argument('--trace-to', nargs="*", default=[], + help="Where to trace (e.g perfetto, perfetto:path, json:log, json:path)") basic_group.add_argument('--storage-path', action="store", type=pathlib.Path, metavar="PATH", help="Location for persisted storage of instance") basic_group.add_argument('--logs-path', action="store", type=pathlib.Path, metavar="PATH", help="Location for test logs") @@ -1051,44 +1056,49 @@ def default_matter_test_main(argv=None, **kwargs): matter_test_config.maximize_cert_chains = kwargs["maximize_cert_chains"] stack = MatterStackState(matter_test_config) - test_config.user_params["matter_stack"] = stash_globally(stack) - # TODO: Steer to right FabricAdmin! - # TODO: If CASE Admin Subject is a CAT tag range, then make sure to issue NOC with that CAT tag + with TracingContext() as tracing_ctx: + for destination in matter_test_config.trace_to: + tracing_ctx.StartFromString(destination) + + test_config.user_params["matter_stack"] = stash_globally(stack) + + # TODO: Steer to right FabricAdmin! + # TODO: If CASE Admin Subject is a CAT tag range, then make sure to issue NOC with that CAT tag - default_controller = stack.certificate_authorities[0].adminList[0].NewController( - nodeId=matter_test_config.controller_node_id, - paaTrustStorePath=str(matter_test_config.paa_trust_store_path), - catTags=matter_test_config.controller_cat_tags - ) - test_config.user_params["default_controller"] = stash_globally(default_controller) + default_controller = stack.certificate_authorities[0].adminList[0].NewController( + nodeId=matter_test_config.controller_node_id, + paaTrustStorePath=str(matter_test_config.paa_trust_store_path), + catTags=matter_test_config.controller_cat_tags + ) + test_config.user_params["default_controller"] = stash_globally(default_controller) - test_config.user_params["matter_test_config"] = stash_globally(matter_test_config) + test_config.user_params["matter_test_config"] = stash_globally(matter_test_config) - test_config.user_params["certificate_authority_manager"] = stash_globally(stack.certificate_authority_manager) + test_config.user_params["certificate_authority_manager"] = stash_globally(stack.certificate_authority_manager) - # Execute the test class with the config - ok = True + # Execute the test class with the config + ok = True - runner = TestRunner(log_dir=test_config.log_path, - testbed_name=test_config.testbed_name) + runner = TestRunner(log_dir=test_config.log_path, + testbed_name=test_config.testbed_name) - with runner.mobly_logger(): - if matter_test_config.commissioning_method is not None: - runner.add_test_class(test_config, CommissionDeviceTest, None) + with runner.mobly_logger(): + if matter_test_config.commissioning_method is not None: + runner.add_test_class(test_config, CommissionDeviceTest, None) - # Add the tests selected unless we have a commission-only request - if not matter_test_config.commission_only: - runner.add_test_class(test_config, test_class, tests) + # Add the tests selected unless we have a commission-only request + if not matter_test_config.commission_only: + runner.add_test_class(test_config, test_class, tests) - try: - runner.run() - ok = runner.results.is_all_pass and ok - except signals.TestAbortAll: - ok = False - except Exception: - logging.exception('Exception when executing %s.', test_config.testbed_name) - ok = False + try: + runner.run() + ok = runner.results.is_all_pass and ok + except signals.TestAbortAll: + ok = False + except Exception: + logging.exception('Exception when executing %s.', test_config.testbed_name) + ok = False # Shutdown the stack when all done stack.Shutdown()