MPI Standalone Implementation - First Bits

.github/workflows/release.yml

@@ -32,3 +32,11 @@ jobs:
           context: .
           tags: faasm/faabric:${{ env.TAG_VERSION }}
           build-args: FAABRIC_VERSION=${{ env.TAG_VERSION }}
+      - name: "Build native MPI container"
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          file: docker/faabric-mpi-native.dockerfile
+          context: .
+          tags: faasm/faabric-mpi-native:${{ env.TAG_VERSION }}
+          build-args: FAABRIC_VERSION=${{ env.TAG_VERSION }}

.github/workflows/tests.yml

@@ -105,7 +105,31 @@ jobs:
         run: inv dev.cc faabric --shared
       - name: "Install Faabric shared library"
         run: inv dev.install faabric --shared
+      - name: "Build MPI native library"
+        run: inv dev.cc faabricmpi_native --shared
+      - name: "Install MPI native library"
+        run: inv dev.install faabricmpi_native --shared
       - name: "Build examples"
         run: inv examples
       - name: "Run example to check"
         run: inv examples.execute check
+
+  mpi_native:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    env:
+      HOST_TYPE: ci
+      REDIS_QUEUE_HOST: redis
+      REDIS_STATE_HOST: redis
+    services:
+      redis:
+        image: redis
+    steps:
+      # --- Code update ---
+      - name: "Fetch ref"
+        uses: actions/checkout@v2
+      # --- Examples ---
+      - name: "Build docker compose images"
+        run: ./mpi-native/build.sh
+      - name: "Run a sample example"
+        run: ./mpi-native/run.sh

CMakeLists.txt

@@ -56,6 +56,7 @@ endfunction()
 add_subdirectory(src/endpoint)
 add_subdirectory(src/executor)
 add_subdirectory(src/mpi)
+add_subdirectory(src/mpi_native)
 add_subdirectory(src/proto)
 add_subdirectory(src/redis)
 add_subdirectory(src/scheduler)
@@ -82,7 +83,7 @@ add_library(faabric
     $<TARGET_OBJECTS:util_obj>
 )
 
-add_dependencies(faabric pistache_ext spdlog_ext)
+add_dependencies(faabric pistache spdlog_ext)
 
 target_link_libraries(faabric
     ${Protobuf_LIBRARIES}

README.md

@@ -110,3 +110,9 @@ inv container.push
 inv container.build --push
 ```
 
+## Additional documentation
+
+More detail on some key features and implementations can be found below:
+
+- [Native MPI builds](docs/native_mpi.md) - run native applications against
+Faabric's MPI library.

cmake/ExternalProjects.cmake

@@ -31,12 +31,18 @@ set(GRPC_PLUGIN /usr/local/bin/grpc_cpp_plugin)
 
 # Pistache 
 ExternalProject_Add(pistache_ext
-    GIT_REPOSITORY "https://github.com/oktal/pistache.git"
+    GIT_REPOSITORY "https://github.com/pistacheio/pistache.git"
     GIT_TAG "2ef937c434810858e05d446e97acbdd6cc1a5a36"
     CMAKE_CACHE_ARGS "-DCMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX}"
+    BUILD_BYPRODUCTS ${CMAKE_INSTALL_PREFIX}/lib/libpistache.so
 )
 ExternalProject_Get_Property(pistache_ext SOURCE_DIR)
 set(PISTACHE_INCLUDE_DIR ${SOURCE_DIR}/include)
+add_library(pistache SHARED IMPORTED)
+add_dependencies(pistache pistache_ext)
+set_target_properties(pistache
+    PROPERTIES IMPORTED_LOCATION ${CMAKE_INSTALL_PREFIX}/lib/libpistache.so
+)
 
 # RapidJSON
 ExternalProject_Add(rapidjson_ext

docker/faabric-mpi-native.dockerfile

@@ -0,0 +1,29 @@
+FROM faasm/grpc-root:0.0.16
+ARG FAABRIC_VERSION
+
+# Note - the version of grpc-root here can be quite behind as it's rebuilt very
+# rarely
+
+# Redis
+RUN apt install -y \
+    libpython3-dev \
+    python3-dev \
+    python3-pip \
+    python3-venv \
+    redis-tools
+
+# Put the code in place
+WORKDIR /code
+RUN git clone -b v${FAABRIC_VERSION} https://github.com/faasm/faabric
+
+WORKDIR /code/faabric
+
+RUN pip3 install invoke
+
+# Build MPI native lib
+RUN inv dev.cmake --shared
+RUN inv dev.cc faabricmpi_native --shared
+RUN inv dev.install faabricmpi_native --shared
+
+# Build examples
+RUN inv examples.build

docs/native_mpi.md

@@ -0,0 +1,20 @@
+# Native MPI execution in Faabric
+
+Faabric supports linking MPI binaries against our custom MPI implementation
+used in [Faasm](https://github.com/faasm/faasm). This way, you can test the
+compliance of your MPI application with our API (a subset of the standard)
+without the burden of cross-compiling to WebAssembly.
+
+To run native MPI applications you need to first modify your binary matching
+the examples provided, and then build the worker image running:
+```
+inv container.build-mpi-native
+```
+
+Then you may run arbitrary deployments setting the right values in
+`mpi-native/mpi-native.env` and running `./mpi-native/run.sh`.
+
+You may remove all stopped and running container images with:
+```bash
+./mpi-native/clean.sh
+```

examples/CMakeLists.txt

@@ -17,6 +17,7 @@ function(add_example example_name)
     target_link_libraries(${example_name}
         faabric
         faabricmpi
+        faabricmpi_native
         protobuf
         pthread
         pistache
@@ -31,6 +32,7 @@ function(add_example example_name)
 endfunction()
 
 add_example(check)
+add_example(mpi_helloworld)
 add_example(server)
 
 add_custom_target(all_examples DEPENDS ${ALL_EXAMPLES})

examples/mpi_helloworld.cpp

@@ -0,0 +1,62 @@
+#include <faabric/util/logging.h>
+
+#include <faabric/mpi-native/MpiExecutor.h>
+#include <faabric/mpi/mpi.h>
+
+int main(int argc, char** argv)
+{
+    auto logger = faabric::util::getLogger();
+    auto& scheduler = faabric::scheduler::getScheduler();
+    auto& conf = faabric::util::getSystemConfig();
+
+    // Global configuration
+    conf.maxNodes = 1;
+    conf.maxNodesPerFunction = 1;
+
+    bool __isRoot;
+    int __worldSize;
+    if (argc < 2) {
+        logger->debug("Non-root process started");
+        __isRoot = false;
+    } else if (argc < 3) {
+        logger->error("Root process started without specifying world size!");
+        return 1;
+    } else {
+        logger->debug("Root process started");
+        __worldSize = std::stoi(argv[2]);
+        __isRoot = true;
+        logger->debug("MPI World Size: {}", __worldSize);
+    }
+
+    // Pre-load message to bootstrap execution
+    if (__isRoot) {
+        faabric::Message msg = faabric::util::messageFactory("mpi", "exec");
+        msg.set_mpiworldsize(__worldSize);
+        scheduler.callFunction(msg);
+    }
+
+    {
+        faabric::executor::SingletonPool p;
+        p.startPool();
+    }
+
+    return 0;
+}
+
+int faabric::executor::mpiFunc()
+{
+    auto logger = faabric::util::getLogger();
+    logger->info("Hello world from Faabric MPI Main!");
+
+    MPI_Init(NULL, NULL);
+
+    int rank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+
+    logger->info("Hello faabric from process {} of {}", rank + 1, worldSize);
+
+    MPI_Finalize();
+
+    return 0;
+}

include/faabric/executor/FaabricExecutor.h

@@ -40,6 +40,8 @@ class FaabricExecutor
                                bool success,
                                const std::string& errorMsg);
 
+    virtual void postFinishCall();
+
     virtual void postFinish();
 
     bool _isBound = false;

include/faabric/executor/FaabricPool.h

@@ -14,7 +14,7 @@ class FaabricPool
 
     void startFunctionCallServer();
 
-    void startThreadPool();
+    void startThreadPool(bool background = true);
 
     void startStateServer();
 
@@ -42,6 +42,14 @@ class FaabricPool
     std::thread poolThread;
     std::vector<std::thread> poolThreads;
 };
+
+class ExecutorPoolFinishedException : public faabric::util::FaabricException
+{
+  public:
+    explicit ExecutorPoolFinishedException(std::string message)
+      : FaabricException(std::move(message))
+    {}
+};
 }
 
 // Macro for quickly defining functions

include/faabric/mpi-native/MpiExecutor.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <faabric/endpoint/FaabricEndpoint.h>
+#include <faabric/executor/FaabricExecutor.h>
+#include <faabric/executor/FaabricPool.h>
+#include <faabric/scheduler/Scheduler.h>
+#include <faabric/util/logging.h>
+
+namespace faabric::executor {
+class MpiExecutor final : public faabric::executor::FaabricExecutor
+{
+  public:
+    explicit MpiExecutor();
+
+    bool doExecute(faabric::Message& msg) override;
+
+    void postFinishCall() override;
+
+    void postFinish() override;
+};
+
+class SingletonPool : public faabric::executor::FaabricPool
+{
+  public:
+    SingletonPool();
+
+    ~SingletonPool();
+
+    void startPool();
+
+  protected:
+    std::unique_ptr<FaabricExecutor> createExecutor(int threadIdx) override
+    {
+        return std::make_unique<MpiExecutor>();
+    }
+
+  private:
+    faabric::endpoint::FaabricEndpoint endpoint;
+    faabric::scheduler::Scheduler& scheduler;
+};
+
+extern faabric::Message* executingCall;
+extern int __attribute__((weak)) mpiFunc();
+}

mpi-native/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+THIS_DIR=$(dirname $(readlink -f $0))
+PROJ_ROOT=${THIS_DIR}/..
+
+pushd ${PROJ_ROOT} >> /dev/null
+
+source ./mpi-native/mpi-native.env
+
+docker-compose \
+    --file ${COMPOSE_FILE} \
+    --env-file ${ENV_FILE} \
+    build
+
+popd >> /dev/null

mpi-native/clean.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+THIS_DIR=$(dirname $(readlink -f $0))
+PROJ_ROOT=${THIS_DIR}/..
+
+pushd ${PROJ_ROOT} >> /dev/null
+
+source ./mpi-native/mpi-native.env
+
+docker-compose \
+    --file ${COMPOSE_FILE} \
+    --env-file ${ENV_FILE} \
+    down
+
+popd >> /dev/null

mpi-native/mpi-native-docker-compose.yml

@@ -0,0 +1,34 @@
+version: "3"
+
+services:
+  redis:
+    image: redis
+
+  worker:
+    image: ${FAABRIC_MPI_NATIVE_IMAGE}
+    entrypoint: /code/faabric/examples/build/${MPI_EXAMPLE}
+    working_dir: /code/faabric
+    privileged: true
+    environment:
+      - LD_LIBRARY_PATH=/usr/local/lib:/build/faabric/install/lib
+      - FUNCTION_STORAGE=local
+      - LOG_LEVEL=debug
+      - REDIS_STATE_HOST=redis
+      - REDIS_QUEUE_HOST=redis
+    depends_on:
+      - redis
+
+  master:
+    image: ${FAABRIC_MPI_NATIVE_IMAGE}
+    entrypoint: ['/code/faabric/examples/build/${MPI_EXAMPLE}', 'master', '${MPI_WORLD_SIZE}']
+    working_dir: /code/faabric
+    privileged: true
+    environment:
+      - LD_LIBRARY_PATH=/usr/local/lib:/build/faabric/install/lib
+      - FUNCTION_STORAGE=local
+      - LOG_LEVEL=debug
+      - REDIS_STATE_HOST=redis
+      - REDIS_QUEUE_HOST=redis
+    depends_on:
+      - redis
+      - worker

mpi-native/mpi-native.env

@@ -0,0 +1,10 @@
+FAABRIC_VERSION=0.0.16
+FAABRIC_MPI_NATIVE_IMAGE=faasm/faabric-mpi-native:0.0.16
+COMPOSE_PROJECT_NAME=faabric-mpi
+
+COMPOSE_FILE="./mpi-native/mpi-native-docker-compose.yml"
+ENV_FILE="./mpi-native/mpi-native.env"
+
+# Deployment-specific
+MPI_WORLD_SIZE=3
+MPI_EXAMPLE=mpi_helloworld