From 92fd7ad0f9993768f34577f8e955ab320b4125a2 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Tue, 13 Feb 2018 12:11:06 -0700
Subject: [PATCH] updated watcher to verify slices are processed with no errors

---
 lib/cluster/execution_controller/engine.js | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/lib/cluster/execution_controller/engine.js b/lib/cluster/execution_controller/engine.js
index 30a9c200732..350b63f72ac 100644
--- a/lib/cluster/execution_controller/engine.js
+++ b/lib/cluster/execution_controller/engine.js
@@ -446,21 +446,33 @@ module.exports = function module(context, messaging, exStore, stateStore, execut
     function _checkAndUpdateExecutionState() {
         let watchDogSet = false;
         let errorCount;
+        let processedCount;
         let watcher;
 
         return () => {
             if (!watchDogSet) {
                 watchDogSet = true;
-                errorCount = executionAnalytics.getAnalytics().failed;
+                const analyticsData = executionAnalytics.getAnalytics();
+                // keep track of how many slices have been processed and failed
+                errorCount = analyticsData.failed;
+                processedCount = analyticsData.processed;
                 _setFailingStatus();
 
                 watcher = setInterval(() => {
-                    const currentErrorCount = executionAnalytics.getAnalytics().failed;
-                    if (errorCount === currentErrorCount) {
+                    const currentAnalyticsData = executionAnalytics.getAnalytics();
+                    const currentErrorCount = currentAnalyticsData.failed;
+                    const currentProcessedCount = currentAnalyticsData.processed;
+                    const errorCountTheSame = currentErrorCount === errorCount;
+                    const slicesHaveProcessedSinceError = currentProcessedCount > processedCount;
+
+                    if (errorCountTheSame && slicesHaveProcessedSinceError) {
                         clearInterval(watcher);
                         logger.info(`No slice errors have occurred within execution: ${exId} will be set back to 'running' state`);
                         exStore.setStatus(exId, 'running');
+                        return;
                     }
+                    errorCount = currentErrorCount;
+                    processedCount = currentProcessedCount;
                 }, executionConfig.probation_window);
             }
         };