Terminate workers if caliper manager is terminated prematurely (#1514)

Caliper can run workers in multiple ways. THe most common is when they are part of the Caliper manager process, however when they are separate processes either on the same machine or remote machines, if the caliper manager is terminated unexpectedly (eg CTRL-C) then those workers still continue to drive load. We would want those workers to terminate as well. This change facilitates this by detecting if the manager is terminated prematurely and sending requests to the workers to terminate This is a resubmitted PR of a collaboration between myself and @CaptainIRS Signed-off-by: Dave Kelsey <[email protected]>
hyperledger-caliper · Feb 28, 2024 · 26da2f2 · 26da2f2
1 parent 8cef10c
commit 26da2f2
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 4 deletions.
diff --git a/packages/caliper-cli/lib/launch/lib/launchManager.js b/packages/caliper-cli/lib/launch/lib/launchManager.js
@@ -59,6 +59,25 @@ class LaunchManager {
                 sutType, Constants.Factories.Connector, require);
 
             const engine = new CaliperEngine(benchmarkConfig, networkConfig, connectorFactory);
+            process.on('SIGINT', async () => {
+                // support a force stop
+                if (!this.terminationAlreadyRequested) {
+                    this.terminationAlreadyRequested = true;
+                    logger.info('Request to terminate received, stopping workers');
+                    try {
+                        await engine.stop();
+                    } catch(error) {
+                        logger.error(`Failed to stop workers: ${error}`);
+                        process.exit(1);
+                    }
+                    logger.info('Workers terminated');
+                    process.exit(0);
+                } else {
+                    logger.info('Termination already requested, terminating immediately');
+                    process.exit(2);
+                }
+            });
+
             const response = await engine.run();
 
             if (response === 0) {

diff --git a/packages/caliper-core/lib/manager/caliper-engine.js b/packages/caliper-core/lib/manager/caliper-engine.js
@@ -180,6 +180,15 @@ class CaliperEngine {
 
         return this.returnCode;
     }
+
+    /**
+     * Stops the benchmark run
+     */
+    async stop() {
+        if (this.roundOrchestrator) {
+            await this.roundOrchestrator.stop();
+        }
+    }
 }
 
 module.exports = CaliperEngine;
diff --git a/packages/caliper-core/lib/manager/orchestrators/round-orchestrator.js b/packages/caliper-core/lib/manager/orchestrators/round-orchestrator.js
@@ -225,14 +225,23 @@ class RoundOrchestrator {
             }
         }
 
-        // clean up, with "silent" failure handling
         try {
             this.report.printResultsByRound();
             await this.report.finalize();
         } catch (err) {
             logger.error(`Error while finalizing the report: ${err.stack || err}`);
         }
 
+        await this.stop();
+        let benchEndTime = Date.now();
+        logger.info(`Benchmark finished in ${(benchEndTime - benchStartTime)/1000.0} seconds. Total rounds: ${success + failed}. Successful rounds: ${success}. Failed rounds: ${failed}.`);
+    }
+
+    /**
+     * Stops the benchmark.
+     */
+    async stop() {
+        // clean up, with "silent" failure handling
         try {
             await this.monitorOrchestrator.stopAllMonitors();
         } catch (err) {
@@ -244,9 +253,6 @@ class RoundOrchestrator {
         } catch (err) {
             logger.error(`Error while stopping workers: ${err.stack || err}`);
         }
-
-        let benchEndTime = Date.now();
-        logger.info(`Benchmark finished in ${(benchEndTime - benchStartTime)/1000.0} seconds. Total rounds: ${success + failed}. Successful rounds: ${success}. Failed rounds: ${failed}.`);
     }
 }