[chip-tool] Add an option to continue running tests even after a test…

… failure
project-chip · Jul 19, 2022 · aa5831c · aa5831c
1 parent fcd6594
commit aa5831c
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 0 deletions.
diff --git a/examples/chip-tool/commands/tests/TestCommand.cpp b/examples/chip-tool/commands/tests/TestCommand.cpp
@@ -77,6 +77,32 @@ void TestCommand::ExitAsync(intptr_t context)
 
 void TestCommand::Exit(std::string message, CHIP_ERROR err)
 {
+    bool shouldContinueOnFailure = mContinueOnFailure.HasValue() && mContinueOnFailure.Value();
+    if (shouldContinueOnFailure)
+    {
+        if (CHIP_NO_ERROR != err)
+        {
+            ChipLogError(chipTool, " ***** Step Failure: %s\n", message.c_str());
+            mErrorMessages.push_back(message);
+            ContinueOnChipMainThread(CHIP_NO_ERROR);
+            return;
+        }
+
+        // If the test runner has been configured to not stop after a test failure, exit can be called with a success but it could
+        // be pending errors from previous steps.
+        if (mErrorMessages.size())
+        {
+            ChipLogError(chipTool, "Error: %lu error(s) has been encountered:", mErrorMessages.size());
+
+            uint32_t errorIndex = 1;
+            for (const auto & errorMessage : mErrorMessages)
+            {
+                ChipLogError(chipTool, "\t%u. %s", errorIndex++, errorMessage.c_str());
+            }
+            err = CHIP_ERROR_INTERNAL;
+        }
+    }
+
     mContinueProcessing = false;
 
     LogEnd(message, err);

diff --git a/examples/chip-tool/commands/tests/TestCommand.h b/examples/chip-tool/commands/tests/TestCommand.h
@@ -49,6 +49,8 @@ class TestCommand : public TestRunner,
         TestRunner(commandName, testsCount), CHIPCommand(commandName, credsIssuerConfig),
         mOnDeviceConnectedCallback(OnDeviceConnectedFn, this), mOnDeviceConnectionFailureCallback(OnDeviceConnectionFailureFn, this)
     {
+        AddArgument("continueOnFailure", 0, 1, &mContinueOnFailure,
+                    "Boolean indicating if the test runner should continue execution if a test fails. Default to false.");
         AddArgument("delayInMs", 0, UINT64_MAX, &mDelayInMs);
         AddArgument("PICS", &mPICSFilePath);
     }
@@ -100,4 +102,8 @@ class TestCommand : public TestRunner,
     // as it still used by the stack afterward. So a task is scheduled to run to close the
     // test suite as soon as possible, and pending events are ignored in between.
     bool mContinueProcessing = true;
+
+    // When set to true, the test runner continue to run after a test failure.
+    chip::Optional<bool> mContinueOnFailure;
+    std::vector<std::string> mErrorMessages;
 };