From 6be03888e139b5be7d91dd9c21b05ee5d9fdf72d Mon Sep 17 00:00:00 2001
From: Carlos Segarra <carlos@carlossegarra.com>
Date: Tue, 30 Nov 2021 17:11:18 +0000
Subject: [PATCH] allow setting the scheduling policy from an environment
 variable

---
 src/scheduler/MpiWorld.cpp      | 10 +++++++---
 src/util/config.cpp             |  2 ++
 tests/test/util/test_config.cpp |  4 ++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/scheduler/MpiWorld.cpp b/src/scheduler/MpiWorld.cpp
index 3d299df2a..fd41ab223 100644
--- a/src/scheduler/MpiWorld.cpp
+++ b/src/scheduler/MpiWorld.cpp
@@ -208,10 +208,14 @@ void MpiWorld::create(faabric::Message& call, int newId, int newSize)
     std::vector<std::string> executedAt;
     if (size > 1) {
         // Send the init messages (note that message i corresponds to rank i+1)
-        // By default, we use the NEVER_ALONE policy for MPI executions to
+        // If enabled, we use the NEVER_ALONE policy for MPI executions to
         // minimise cross-host messaging
-        faabric::util::SchedulingDecision decision = sch.callFunctions(
-          req, faabric::util::SchedulingTopologyHint::NEVER_ALONE);
+        auto topologyHint = faabric::util::SchedulingTopologyHint::NORMAL;
+        if (faabric::util::getSystemConfig().useTopologyHints == "on") {
+            topologyHint = faabric::util::SchedulingTopologyHint::NEVER_ALONE;
+        }
+        faabric::util::SchedulingDecision decision =
+          sch.callFunctions(req, topologyHint);
         executedAt = decision.hosts;
     }
     assert(executedAt.size() == size - 1);
diff --git a/src/util/config.cpp b/src/util/config.cpp
index 633291de3..0f7b9f067 100644
--- a/src/util/config.cpp
+++ b/src/util/config.cpp
@@ -34,6 +34,7 @@ void SystemConfig::initialise()
     // Scheduling
     noScheduler = this->getSystemConfIntParam("NO_SCHEDULER", "0");
     overrideCpuCount = this->getSystemConfIntParam("OVERRIDE_CPU_COUNT", "0");
+    useTopologyHints = getEnvVar("USE_TOPOLOGY_HINTS", "off");
 
     // Worker-related timeouts (all in seconds)
     globalMessageTimeout =
@@ -100,6 +101,7 @@ void SystemConfig::print()
     SPDLOG_INFO("--- Scheduling ---");
     SPDLOG_INFO("NO_SCHEDULER               {}", noScheduler);
     SPDLOG_INFO("OVERRIDE_CPU_COUNT         {}", overrideCpuCount);
+    SPDLOG_INFO("USE_TOPOLOGY_HINTS         {}", useTopologyHints);
 
     SPDLOG_INFO("--- Timeouts ---");
     SPDLOG_INFO("GLOBAL_MESSAGE_TIMEOUT     {}", globalMessageTimeout);
diff --git a/tests/test/util/test_config.cpp b/tests/test/util/test_config.cpp
index 1c734758f..22fe6a7c3 100644
--- a/tests/test/util/test_config.cpp
+++ b/tests/test/util/test_config.cpp
@@ -22,6 +22,7 @@ TEST_CASE("Test default system config initialisation", "[util]")
 
     REQUIRE(conf.noScheduler == 0);
     REQUIRE(conf.overrideCpuCount == 0);
+    REQUIRE(conf.useTopologyHints == "off");
 
     REQUIRE(conf.globalMessageTimeout == 60000);
     REQUIRE(conf.boundTimeout == 30000);
@@ -44,6 +45,7 @@ TEST_CASE("Test overriding system config initialisation", "[util]")
 
     std::string noScheduler = setEnvVar("NO_SCHEDULER", "1");
     std::string overrideCpuCount = setEnvVar("OVERRIDE_CPU_COUNT", "4");
+    std::string useTopologyHints = setEnvVar("USE_TOPOLOGY_HINTS", "on");
 
     std::string globalTimeout = setEnvVar("GLOBAL_MESSAGE_TIMEOUT", "9876");
     std::string boundTimeout = setEnvVar("BOUND_TIMEOUT", "6666");
@@ -70,6 +72,7 @@ TEST_CASE("Test overriding system config initialisation", "[util]")
 
     REQUIRE(conf.noScheduler == 1);
     REQUIRE(conf.overrideCpuCount == 4);
+    REQUIRE(conf.useTopologyHints == "on");
 
     REQUIRE(conf.globalMessageTimeout == 9876);
     REQUIRE(conf.boundTimeout == 6666);
@@ -95,6 +98,7 @@ TEST_CASE("Test overriding system config initialisation", "[util]")
 
     setEnvVar("NO_SCHEDULER", noScheduler);
     setEnvVar("OVERRIDE_CPU_COUNT", overrideCpuCount);
+    setEnvVar("USE_TOPOLOGY_HINTS", useTopologyHints);
 
     setEnvVar("GLOBAL_MESSAGE_TIMEOUT", globalTimeout);
     setEnvVar("BOUND_TIMEOUT", boundTimeout);