updated watcher to verify slices are processed with no errors

terascope · Feb 13, 2018 · 92fd7ad · 92fd7ad
1 parent 951838e
commit 92fd7ad
Showing 1 changed file with 15 additions and 3 deletions.
diff --git a/lib/cluster/execution_controller/engine.js b/lib/cluster/execution_controller/engine.js
@@ -446,21 +446,33 @@ module.exports = function module(context, messaging, exStore, stateStore, execut
     function _checkAndUpdateExecutionState() {
         let watchDogSet = false;
         let errorCount;
+        let processedCount;
         let watcher;
 
         return () => {
             if (!watchDogSet) {
                 watchDogSet = true;
-                errorCount = executionAnalytics.getAnalytics().failed;
+                const analyticsData = executionAnalytics.getAnalytics();
+                // keep track of how many slices have been processed and failed
+                errorCount = analyticsData.failed;
+                processedCount = analyticsData.processed;
                 _setFailingStatus();
 
                 watcher = setInterval(() => {
-                    const currentErrorCount = executionAnalytics.getAnalytics().failed;
-                    if (errorCount === currentErrorCount) {
+                    const currentAnalyticsData = executionAnalytics.getAnalytics();
+                    const currentErrorCount = currentAnalyticsData.failed;
+                    const currentProcessedCount = currentAnalyticsData.processed;
+                    const errorCountTheSame = currentErrorCount === errorCount;
+                    const slicesHaveProcessedSinceError = currentProcessedCount > processedCount;
+
+                    if (errorCountTheSame && slicesHaveProcessedSinceError) {
                         clearInterval(watcher);
                         logger.info(`No slice errors have occurred within execution: ${exId} will be set back to 'running' state`);
                         exStore.setStatus(exId, 'running');
+                        return;
                     }
+                    errorCount = currentErrorCount;
+                    processedCount = currentProcessedCount;
                 }, executionConfig.probation_window);
             }
         };