From 7fb7eaba4123441af4e2627abe276f6f893fc5be Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 22 Nov 2021 10:51:32 +0000 Subject: [PATCH 1/5] add utility function to return the set of distinct hosts in an exec graph --- include/faabric/scheduler/ExecGraph.h | 5 +++++ src/scheduler/ExecGraph.cpp | 22 ++++++++++++++++++++++ tests/test/scheduler/test_exec_graph.cpp | 19 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/include/faabric/scheduler/ExecGraph.h b/include/faabric/scheduler/ExecGraph.h index 6f02866ef..aaf45daca 100644 --- a/include/faabric/scheduler/ExecGraph.h +++ b/include/faabric/scheduler/ExecGraph.h @@ -1,9 +1,12 @@ #pragma once #include + +#include #include namespace faabric::scheduler { + struct ExecGraphNode { faabric::Message msg; @@ -17,6 +20,8 @@ struct ExecGraph int countExecGraphNodes(const ExecGraph& graph); +std::set countExecGraphHosts(const ExecGraph& graph); + std::string execNodeToJson(const ExecGraphNode& node); std::string execGraphToJson(const ExecGraph& graph); diff --git a/src/scheduler/ExecGraph.cpp b/src/scheduler/ExecGraph.cpp index 52c6fbd21..559856304 100644 --- a/src/scheduler/ExecGraph.cpp +++ b/src/scheduler/ExecGraph.cpp @@ -25,6 +25,28 @@ int countExecGraphNodes(const ExecGraph& graph) return count; } +std::set countExecGraphHostsForNode(const ExecGraphNode& node) +{ + std::set hostsForNode; + hostsForNode.insert(node.msg.executedhost()); + + if (!node.children.empty()) { + for (auto c : node.children) { + std::set nodeHost = countExecGraphHostsForNode(c); + hostsForNode.insert(nodeHost.begin(), nodeHost.end()); + } + } + + return hostsForNode; +} + +std::set countExecGraphHosts(const ExecGraph& graph) +{ + ExecGraphNode rootNode = graph.rootNode; + std::set hosts = countExecGraphHostsForNode(rootNode); + return hosts; +} + // ---------------------------------------- // TODO - do this with RapidJson and not sstream // ---------------------------------------- diff --git a/tests/test/scheduler/test_exec_graph.cpp b/tests/test/scheduler/test_exec_graph.cpp index b1619b171..51c324463 100644 --- a/tests/test/scheduler/test_exec_graph.cpp +++ b/tests/test/scheduler/test_exec_graph.cpp @@ -72,6 +72,25 @@ TEST_CASE("Test execution graph", "[scheduler][exec-graph]") checkExecGraphEquality(expected, actual); } +TEST_CASE("Test execution graph node count", "[scheduler][exec-graph]") +{ + faabric::Message msgA = faabric::util::messageFactory("demo", "echo"); + faabric::Message msgB1 = faabric::util::messageFactory("demo", "echo"); + faabric::Message msgB2 = faabric::util::messageFactory("demo", "echo"); + + msgA.set_executedhost("foo"); + msgB1.set_executedhost("bar"); + msgB2.set_executedhost("baz"); + + ExecGraphNode nodeB2 = { .msg = msgB2 }; + ExecGraphNode nodeB1 = { .msg = msgB1 }; + ExecGraphNode nodeA = { .msg = msgA, .children = { nodeB1, nodeB2 } }; + + ExecGraph graph{ .rootNode = nodeA }; + auto hosts = faabric::scheduler::countExecGraphHosts(graph); + REQUIRE(hosts.size() == 3); +} + TEST_CASE_METHOD(MpiBaseTestFixture, "Test MPI execution graph", "[mpi][scheduler][exec-graph]") From 142e513457884f80841e481ac191bc5f08c0fe42 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 22 Nov 2021 10:52:39 +0000 Subject: [PATCH 2/5] bump version --- .env | 4 ++-- .github/workflows/tests.yml | 6 +++--- VERSION | 2 +- mpi-native/mpi-native.env | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 55f963e83..e06034e91 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ -FAABRIC_VERSION=0.2.1 -FAABRIC_CLI_IMAGE=faasm/faabric:0.2.1 +FAABRIC_VERSION=0.2.2 +FAABRIC_CLI_IMAGE=faasm/faabric:0.2.2 COMPOSE_PROJECT_NAME=faabric-dev diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 908905126..59ebfbfd0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm/faabric:0.2.1 + image: faasm/faabric:0.2.2 defaults: run: working-directory: /code/faabric @@ -45,7 +45,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm/faabric:0.2.1 + image: faasm/faabric:0.2.2 defaults: run: working-directory: /code/faabric @@ -92,7 +92,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: faasm/faabric:0.2.1 + image: faasm/faabric:0.2.2 defaults: run: working-directory: /code/faabric diff --git a/VERSION b/VERSION index 0c62199f1..ee1372d33 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.1 +0.2.2 diff --git a/mpi-native/mpi-native.env b/mpi-native/mpi-native.env index 5b6f0bc21..a48ea72c9 100644 --- a/mpi-native/mpi-native.env +++ b/mpi-native/mpi-native.env @@ -1,4 +1,4 @@ -FAABRIC_VERSION=0.2.1 +FAABRIC_VERSION=0.2.2 FAABRIC_MPI_NATIVE_IMAGE=faasm/faabric-mpi-native:0.0.18 COMPOSE_PROJECT_NAME=faabric-mpi From ee278b783a06e0b5fb8af2ad2eee4a564662bf78 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 22 Nov 2021 15:28:05 +0000 Subject: [PATCH 3/5] refactor from count to get --- include/faabric/scheduler/ExecGraph.h | 2 +- src/scheduler/ExecGraph.cpp | 8 ++++---- tests/test/scheduler/test_exec_graph.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/faabric/scheduler/ExecGraph.h b/include/faabric/scheduler/ExecGraph.h index aaf45daca..4e3a4631a 100644 --- a/include/faabric/scheduler/ExecGraph.h +++ b/include/faabric/scheduler/ExecGraph.h @@ -20,7 +20,7 @@ struct ExecGraph int countExecGraphNodes(const ExecGraph& graph); -std::set countExecGraphHosts(const ExecGraph& graph); +std::set getExecGraphHosts(const ExecGraph& graph); std::string execNodeToJson(const ExecGraphNode& node); diff --git a/src/scheduler/ExecGraph.cpp b/src/scheduler/ExecGraph.cpp index 559856304..21c945a83 100644 --- a/src/scheduler/ExecGraph.cpp +++ b/src/scheduler/ExecGraph.cpp @@ -25,14 +25,14 @@ int countExecGraphNodes(const ExecGraph& graph) return count; } -std::set countExecGraphHostsForNode(const ExecGraphNode& node) +std::set getExecGraphHostsForNode(const ExecGraphNode& node) { std::set hostsForNode; hostsForNode.insert(node.msg.executedhost()); if (!node.children.empty()) { for (auto c : node.children) { - std::set nodeHost = countExecGraphHostsForNode(c); + std::set nodeHost = getExecGraphHostsForNode(c); hostsForNode.insert(nodeHost.begin(), nodeHost.end()); } } @@ -40,10 +40,10 @@ std::set countExecGraphHostsForNode(const ExecGraphNode& node) return hostsForNode; } -std::set countExecGraphHosts(const ExecGraph& graph) +std::set getExecGraphHosts(const ExecGraph& graph) { ExecGraphNode rootNode = graph.rootNode; - std::set hosts = countExecGraphHostsForNode(rootNode); + std::set hosts = getExecGraphHostsForNode(rootNode); return hosts; } diff --git a/tests/test/scheduler/test_exec_graph.cpp b/tests/test/scheduler/test_exec_graph.cpp index 51c324463..973ffb121 100644 --- a/tests/test/scheduler/test_exec_graph.cpp +++ b/tests/test/scheduler/test_exec_graph.cpp @@ -87,7 +87,7 @@ TEST_CASE("Test execution graph node count", "[scheduler][exec-graph]") ExecGraphNode nodeA = { .msg = msgA, .children = { nodeB1, nodeB2 } }; ExecGraph graph{ .rootNode = nodeA }; - auto hosts = faabric::scheduler::countExecGraphHosts(graph); + auto hosts = faabric::scheduler::getExecGraphHosts(graph); REQUIRE(hosts.size() == 3); } From d21fb97b2961ce5483b2dbbb8f9464ee587e58cb Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 22 Nov 2021 16:41:18 +0000 Subject: [PATCH 4/5] pr comments --- src/scheduler/ExecGraph.cpp | 12 ++++-------- tests/test/scheduler/test_exec_graph.cpp | 11 +++++++---- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/scheduler/ExecGraph.cpp b/src/scheduler/ExecGraph.cpp index 21c945a83..04490aa92 100644 --- a/src/scheduler/ExecGraph.cpp +++ b/src/scheduler/ExecGraph.cpp @@ -30,11 +30,9 @@ std::set getExecGraphHostsForNode(const ExecGraphNode& node) std::set hostsForNode; hostsForNode.insert(node.msg.executedhost()); - if (!node.children.empty()) { - for (auto c : node.children) { - std::set nodeHost = getExecGraphHostsForNode(c); - hostsForNode.insert(nodeHost.begin(), nodeHost.end()); - } + for (auto c : node.children) { + std::set nodeHost = getExecGraphHostsForNode(c); + hostsForNode.insert(nodeHost.begin(), nodeHost.end()); } return hostsForNode; @@ -42,9 +40,7 @@ std::set getExecGraphHostsForNode(const ExecGraphNode& node) std::set getExecGraphHosts(const ExecGraph& graph) { - ExecGraphNode rootNode = graph.rootNode; - std::set hosts = getExecGraphHostsForNode(rootNode); - return hosts; + return getExecGraphHostsForNode(graph.rootNode); } // ---------------------------------------- diff --git a/tests/test/scheduler/test_exec_graph.cpp b/tests/test/scheduler/test_exec_graph.cpp index 973ffb121..44b17ae10 100644 --- a/tests/test/scheduler/test_exec_graph.cpp +++ b/tests/test/scheduler/test_exec_graph.cpp @@ -72,7 +72,7 @@ TEST_CASE("Test execution graph", "[scheduler][exec-graph]") checkExecGraphEquality(expected, actual); } -TEST_CASE("Test execution graph node count", "[scheduler][exec-graph]") +TEST_CASE("Test get unique hosts from exec graph", "[scheduler][exec-graph]") { faabric::Message msgA = faabric::util::messageFactory("demo", "echo"); faabric::Message msgB1 = faabric::util::messageFactory("demo", "echo"); @@ -82,13 +82,16 @@ TEST_CASE("Test execution graph node count", "[scheduler][exec-graph]") msgB1.set_executedhost("bar"); msgB2.set_executedhost("baz"); - ExecGraphNode nodeB2 = { .msg = msgB2 }; ExecGraphNode nodeB1 = { .msg = msgB1 }; - ExecGraphNode nodeA = { .msg = msgA, .children = { nodeB1, nodeB2 } }; + ExecGraphNode nodeB2 = { .msg = msgB2 }; + ExecGraphNode nodeB3 = { .msg = msgB2 }; + ExecGraphNode nodeA = { .msg = msgA, + .children = { nodeB1, nodeB2, nodeB3 } }; ExecGraph graph{ .rootNode = nodeA }; + std::set expected = { "bar", "baz", "foo" }; auto hosts = faabric::scheduler::getExecGraphHosts(graph); - REQUIRE(hosts.size() == 3); + REQUIRE(hosts == expected); } TEST_CASE_METHOD(MpiBaseTestFixture, From aebd37a2337edac1482d981e80300515c497232b Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 22 Nov 2021 18:47:52 +0000 Subject: [PATCH 5/5] bump faabric base --- docker/faabric.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/faabric.dockerfile b/docker/faabric.dockerfile index 2dd092f60..145e4c9f5 100644 --- a/docker/faabric.dockerfile +++ b/docker/faabric.dockerfile @@ -1,4 +1,4 @@ -FROM faasm/faabric-base:0.2.0 +FROM faasm/faabric-base:0.2.2 ARG FAABRIC_VERSION # faabic-base image is not re-built often, so tag may be behind