From f0b6cbcc23e734087939ec533f9aa1ef0f58ecf1 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Mon, 18 Oct 2021 09:05:25 +0200 Subject: [PATCH] [CI] Add an additional delayInMs argument when running a test (#10560) --- .../chip-tool/commands/tests/TestCommand.h | 14 +- .../templates/partials/test_cluster.zapt | 2 + scripts/tests/test_suites.sh | 15 ++- .../chip-tool/zap-generated/test/Commands.h | 120 ++++++++++++++++++ 4 files changed, 144 insertions(+), 7 deletions(-) diff --git a/examples/chip-tool/commands/tests/TestCommand.h b/examples/chip-tool/commands/tests/TestCommand.h index 5670c464b4efb2..0488435456194d 100644 --- a/examples/chip-tool/commands/tests/TestCommand.h +++ b/examples/chip-tool/commands/tests/TestCommand.h @@ -20,6 +20,7 @@ #include "../common/CHIPCommand.h" #include +#include #include class TestCommand : public CHIPCommand @@ -28,7 +29,9 @@ class TestCommand : public CHIPCommand TestCommand(const char * commandName) : CHIPCommand(commandName), mOnDeviceConnectedCallback(OnDeviceConnectedFn, this), mOnDeviceConnectionFailureCallback(OnDeviceConnectionFailureFn, this) - {} + { + AddArgument("delayInMs", 0, UINT64_MAX, &mDelayInMs); + } /////////// CHIPCommand Interface ///////// CHIP_ERROR Run(NodeId remoteId) override; @@ -103,4 +106,13 @@ class TestCommand : public CHIPCommand chip::Callback::Callback mOnDeviceConnectedCallback; chip::Callback::Callback mOnDeviceConnectionFailureCallback; + + void Wait() + { + if (mDelayInMs) + { + chip::test_utils::SleepMillis(mDelayInMs); + } + }; + uint64_t mDelayInMs = 0; }; diff --git a/examples/chip-tool/templates/partials/test_cluster.zapt b/examples/chip-tool/templates/partials/test_cluster.zapt index bbdae1dbede5fa..a1813e758ba11f 100644 --- a/examples/chip-tool/templates/partials/test_cluster.zapt +++ b/examples/chip-tool/templates/partials/test_cluster.zapt @@ -21,6 +21,8 @@ class {{filename}}: public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an diff --git a/scripts/tests/test_suites.sh b/scripts/tests/test_suites.sh index 6747917d33d8d0..b23d5ef0e203bd 100755 --- a/scripts/tests/test_suites.sh +++ b/scripts/tests/test_suites.sh @@ -19,14 +19,16 @@ set -e declare -i iterations=2 +declare -i delay=0 declare -i background_pid=0 declare test_case_wrapper=() usage() { - echo "test_suites.sh [-a APPLICATION] [-i ITERATIONS] [-h] [-s CASE_NAME] [-w COMMAND]" + echo "test_suites.sh [-a APPLICATION] [-i ITERATIONS] [-h] [-s CASE_NAME] [-w COMMAND] [-d DELAY]" echo " -a APPLICATION: runs chip-tool against 'chip--app' (default: all-clusters)" - echo " -i ITERATIONS: number of iterations to run (default: $iterations)" + echo " -d DELAY: milliseconds to wait before running an individual test step (default: $delay)" echo " -h: this help message" + echo " -i ITERATIONS: number of iterations to run (default: $iterations)" echo " -s CASE_NAME: runs single test case name (e.g. Test_TC_OO_2_2" echo " for Test_TC_OO_2_2.yaml) (by default, all are run)" echo " -w COMMAND: prefix all instantiations with a command (e.g. valgrind) (default: '')" @@ -35,11 +37,12 @@ usage() { } # read shell arguments -while getopts a:i:hs:w: flag; do +while getopts a:d:i:hs:w: flag; do case "$flag" in a) application=$OPTARG ;; - i) iterations=$OPTARG ;; + d) delay=$OPTARG ;; h) usage ;; + i) iterations=$OPTARG ;; s) single_case=$OPTARG ;; w) test_case_wrapper=("$OPTARG") ;; esac @@ -61,7 +64,7 @@ if [[ $iterations == 0 ]]; then exit 1 fi -echo "Running tests for application: $application, with iterations set to: $iterations" +echo "Running tests for application: $application, with iterations set to: $iterations and delay set to $delay" cleanup() { if [[ $background_pid != 0 ]]; then @@ -123,7 +126,7 @@ for j in "${iter_array[@]}"; do echo " * Pairing to device" "${test_case_wrapper[@]}" out/debug/standalone/chip-tool pairing qrcode MT:D8XA0CQM00KA0648G00 echo " * Starting test run: $i" - "${test_case_wrapper[@]}" out/debug/standalone/chip-tool tests "$i" + "${test_case_wrapper[@]}" out/debug/standalone/chip-tool tests "$i" "$delay" # Prevent cleanup trying to kill a process we already killed. temp_background_pid=$background_pid background_pid=0 diff --git a/zzz_generated/chip-tool/zap-generated/test/Commands.h b/zzz_generated/chip-tool/zap-generated/test/Commands.h index e79750b7dd8437..950de8eb694acf 100644 --- a/zzz_generated/chip-tool/zap-generated/test/Commands.h +++ b/zzz_generated/chip-tool/zap-generated/test/Commands.h @@ -43,6 +43,8 @@ class TV_TargetNavigatorCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -156,6 +158,8 @@ class TV_AudioOutputCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -292,6 +296,8 @@ class TV_ApplicationLauncherCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -472,6 +478,8 @@ class TV_KeypadInputCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -549,6 +557,8 @@ class TV_AccountLoginCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -655,6 +665,8 @@ class TV_WakeOnLanCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -736,6 +748,8 @@ class TV_ApplicationBasicCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -906,6 +920,8 @@ class TV_MediaPlaybackCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -1324,6 +1340,8 @@ class TV_TvChannelCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -1461,6 +1479,8 @@ class TV_LowPowerCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -1533,6 +1553,8 @@ class TV_MediaInputCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -1751,6 +1773,8 @@ class TestCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -5648,6 +5672,8 @@ class TestConstraints : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -5821,6 +5847,8 @@ class TestDelayCommands : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -5873,6 +5901,8 @@ class TestDescriptorCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -6056,6 +6086,8 @@ class TestSubscribe_OnOff : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -6326,6 +6358,8 @@ class Test_TC_OO_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -6563,6 +6597,8 @@ class Test_TC_OO_2_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -6989,6 +7025,8 @@ class Test_TC_OO_2_2 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -7435,6 +7473,8 @@ class Test_TC_DM_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -8110,6 +8150,8 @@ class Test_TC_DM_3_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -8156,6 +8198,8 @@ class Test_TC_WNCV_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -8299,6 +8343,8 @@ class Test_TC_WNCV_2_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -8696,6 +8742,8 @@ class Test_TC_WNCV_3_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -8825,6 +8873,8 @@ class Test_TC_WNCV_3_2 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -8954,6 +9004,8 @@ class Test_TC_WNCV_3_3 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9083,6 +9135,8 @@ class Test_TC_BI_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9226,6 +9280,8 @@ class Test_TC_FLW_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9305,6 +9361,8 @@ class Test_TC_TM_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9448,6 +9506,8 @@ class Test_TC_OCC_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9559,6 +9619,8 @@ class TestOperationalCredentialsCluster : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9673,6 +9735,8 @@ class Test_TC_LVL_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9752,6 +9816,8 @@ class Test_TC_CC_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9831,6 +9897,8 @@ class Test_TC_RH_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9910,6 +9978,8 @@ class Test_TC_MC_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -9989,6 +10059,8 @@ class Test_TC_TSTAT_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -10068,6 +10140,8 @@ class Test_TC_PCC_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -10147,6 +10221,8 @@ class Test_TC_TSUIC_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -10226,6 +10302,8 @@ class Test_TC_DIAGTH_1_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -10369,6 +10447,8 @@ class Test_TC_TM_2_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -10448,6 +10528,8 @@ class Test_TC_TSUIC_2_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -10987,6 +11069,8 @@ class Test_TC_PCC_2_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -11294,6 +11378,8 @@ class Test_TC_CC_3_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -11583,6 +11669,8 @@ class Test_TC_CC_3_2 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -11868,6 +11956,8 @@ class Test_TC_CC_3_3 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -12093,6 +12183,8 @@ class Test_TC_CC_4_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -12285,6 +12377,8 @@ class Test_TC_CC_4_2 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -12508,6 +12602,8 @@ class Test_TC_CC_4_3 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -12733,6 +12829,8 @@ class Test_TC_CC_4_4 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -12927,6 +13025,8 @@ class Test_TC_CC_5_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -13120,6 +13220,8 @@ class Test_TC_CC_5_2 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -13341,6 +13443,8 @@ class Test_TC_CC_5_3 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -13534,6 +13638,8 @@ class Test_TC_CC_6_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -13727,6 +13833,8 @@ class Test_TC_CC_6_2 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -13990,6 +14098,8 @@ class Test_TC_CC_6_3 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -14221,6 +14331,8 @@ class Test_TC_CC_7_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -14447,6 +14559,8 @@ class Test_TC_CC_7_2 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -14732,6 +14846,8 @@ class Test_TC_CC_7_3 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -14957,6 +15073,8 @@ class Test_TC_CC_7_4 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an @@ -15151,6 +15269,8 @@ class Test_TC_CC_8_1 : public TestCommand return; } + Wait(); + // Ensure we increment mTestIndex before we start running the relevant // command. That way if we lose the timeslice after we send the message // but before our function call returns, we won't end up with an