Merge w/ master upstream

csegarragonz · Dec 15, 2020 · 835cc5d · 835cc5d
1 parent 32e9159
commit 835cc5d
Showing 69 changed files with 213 additions and 157 deletions.
diff --git a/.env b/.env
@@ -1,3 +1,3 @@
-FAASM_VERSION=0.0.16
-CLI_IMAGE=faasm/faabric:0.0.16
+FAABRIC_VERSION=0.0.16
+FAABRIC_CLI_IMAGE=faasm/faabric:0.0.16
 COMPOSE_PROJECT_NAME=faabric-dev
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -13,17 +13,8 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 # Compile comamnds for clang tools
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-# Library funcs
-function(faabric_lib lib_name lib_deps)
-    add_library(${lib_name} ${lib_deps})
-
-    add_library(${lib_name}_obj OBJECT ${lib_deps})
-
-    if(BUILD_SHARED_LIBS)
-        target_compile_options(${lib_name} PRIVATE "-fPIC")
-        target_compile_options(${lib_name}_obj PRIVATE "-fPIC")
-    endif()    
-endfunction()
+# Global include dir
+set(FAABRIC_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/include)
 
 # Output directories
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
@@ -33,11 +24,34 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 # External libraries
 include(cmake/ExternalProjects.cmake)
 
-# Global include dir
-set(FAABRIC_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/include)
+# Library funcs
+function(faabric_lib lib_name lib_deps)
+    # "Normal" library used for linking internally
+    add_library(${lib_name} ${lib_deps})
+    target_include_directories(${lib_name} 
+        PUBLIC ${FAABRIC_INCLUDE_DIR}
+        PUBLIC ${PISTACHE_INCLUDE_DIR}
+        PUBLIC ${SPDLOG_INCLUDE_DIR}
+        PUBLIC ${RAPIDJSON_INCLUDE_DIR}
+        )
+
+    # Object library for bundling everything together (should have the same
+    # include dirs and dependencies as the normal library)
+    add_library(${lib_name}_obj OBJECT ${lib_deps})
+    target_include_directories(${lib_name}_obj 
+        PUBLIC ${FAABRIC_INCLUDE_DIR}
+        PUBLIC ${PISTACHE_INCLUDE_DIR}
+        PUBLIC ${SPDLOG_INCLUDE_DIR}
+        PUBLIC ${RAPIDJSON_INCLUDE_DIR}
+        )
 
-# Include directories
-include_directories(${FAABRIC_INCLUDE_DIR})
+    target_link_libraries(${lib_name}_obj ${lib_name})
+
+    if(BUILD_SHARED_LIBS)
+        target_compile_options(${lib_name} PRIVATE "-fPIC")
+        target_compile_options(${lib_name}_obj PRIVATE "-fPIC")
+    endif()    
+endfunction()
 
 add_subdirectory(src/endpoint)
 add_subdirectory(src/executor)
@@ -48,8 +62,8 @@ add_subdirectory(src/scheduler)
 add_subdirectory(src/state)
 add_subdirectory(src/util)
 
-# Wrapper library - note we want to include all the targets in this library, not
-# just link it superficially
+# Wrapper library - note we want to include all the _object_ targets in this 
+# library to ensure it's all bundled in together
 if(BUILD_SHARED_LIBS) 
     set(FAABRIC_LIB_TYPE SHARED)
 else()
@@ -68,7 +82,7 @@ add_library(faabric
     $<TARGET_OBJECTS:util_obj>
 )
 
-add_dependencies(faabric pistache_ext)
+add_dependencies(faabric pistache_ext spdlog_ext)
 
 target_link_libraries(faabric
     ${Protobuf_LIBRARIES}

diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake
@@ -36,7 +36,7 @@ ExternalProject_Add(pistache_ext
     CMAKE_CACHE_ARGS "-DCMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX}"
 )
 ExternalProject_Get_Property(pistache_ext SOURCE_DIR)
-include_directories(${SOURCE_DIR}/include)
+set(PISTACHE_INCLUDE_DIR ${SOURCE_DIR}/include)
 
 # RapidJSON
 ExternalProject_Add(rapidjson_ext
@@ -48,7 +48,7 @@ ExternalProject_Add(rapidjson_ext
     CMAKE_CACHE_ARGS "-DCMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX}"
 )
 ExternalProject_Get_Property(rapidjson_ext SOURCE_DIR)
-include_directories(${SOURCE_DIR}/include)
+set(RAPIDJSON_INCLUDE_DIR ${SOURCE_DIR}/include)
 
 # spdlog
 ExternalProject_Add(spdlog_ext
@@ -57,7 +57,7 @@ ExternalProject_Add(spdlog_ext
     CMAKE_CACHE_ARGS "-DCMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX}"
 )
 ExternalProject_Get_Property(spdlog_ext SOURCE_DIR)
-include_directories(${SOURCE_DIR}/include)
+set(SPDLOG_INCLUDE_DIR ${SOURCE_DIR}/include)
 
 if(FAABRIC_BUILD_TESTS)
     # Catch (tests)

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -7,8 +7,10 @@ services:
   cli:
     image: ${FAABRIC_CLI_IMAGE}
     volumes:
-        - .:/code/faabric
-        - ./build:/build/faabric
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /usr/bin/docker:/usr/bin/docker
+      - .:/code/faabric
+      - ./build:/build/faabric
     working_dir: /code/faabric
     stdin_open: true
     tty: true

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -19,12 +19,14 @@ function(add_example example_name)
     target_link_libraries(${example_name}
         ${FAABRIC_LIBS}
         pistache
+        protobuf
         )
 
     set(ALL_EXAMPLES ${ALL_EXAMPLES} ${example_name} PARENT_SCOPE)   
 endfunction()
 
 add_example(check)
+add_example(mpi_helloworld)
 add_example(server)
 
 add_custom_target(all_examples DEPENDS ${ALL_EXAMPLES})
diff --git a/examples/mpi_helloworld.cpp b/examples/mpi_helloworld.cpp
@@ -0,0 +1,19 @@
+#include <faabric/mpi/mpi.h>
+#include <faabric/util/logging.h>
+
+int main()
+{
+    auto logger = faabric::util::getLogger();
+
+    MPI_Init(NULL, NULL);
+
+    int rank, worldSize;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
+
+    logger->info("Hello world from rank %i of %i", rank, worldSize);
+
+    MPI_Finalize();
+
+    return 0;
+}
diff --git a/include/faabric/executor/FaabricMain.h b/include/faabric/executor/FaabricMain.h
@@ -1,7 +1,6 @@
 #pragma once
 
-#include "FaabricPool.h"
-
+#include <faabric/executor/FaabricPool.h>
 #include <faabric/state/StateServer.h>
 #include <faabric/util/config.h>
 

diff --git a/include/faabric/scheduler/FunctionCallServer.h b/include/faabric/scheduler/FunctionCallServer.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "Scheduler.h"
+#include <faabric/scheduler/Scheduler.h>
 
 #include <faabric/proto/RPCServer.h>
 #include <faabric/proto/faabric.grpc.pb.h>

diff --git a/include/faabric/scheduler/Scheduler.h b/include/faabric/scheduler/Scheduler.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include "ExecGraph.h"
-#include "InMemoryMessageQueue.h"
+#include <faabric/scheduler/ExecGraph.h>
+#include <faabric/scheduler/InMemoryMessageQueue.h>
 
 #include <faabric/util/config.h>
 #include <faabric/util/func.h>

diff --git a/include/faabric/state/DummyStateServer.h b/include/faabric/state/DummyStateServer.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "StateServer.h"
+#include <faabric/state/StateServer.h>
 
 namespace faabric::state {
 class DummyStateServer

diff --git a/include/faabric/state/InMemoryStateKeyValue.h b/include/faabric/state/InMemoryStateKeyValue.h
@@ -1,8 +1,8 @@
 #pragma once
 
-#include "InMemoryStateRegistry.h"
-#include "StateClient.h"
-#include "StateKeyValue.h"
+#include <faabric/state/InMemoryStateRegistry.h>
+#include <faabric/state/StateClient.h>
+#include <faabric/state/StateKeyValue.h>
 
 #include <faabric/proto/faabric.grpc.pb.h>
 #include <faabric/proto/faabric.pb.h>

diff --git a/include/faabric/state/RedisStateKeyValue.h b/include/faabric/state/RedisStateKeyValue.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "StateKeyValue.h"
+#include <faabric/state/StateKeyValue.h>
 
 #include <faabric/redis/Redis.h>
 #include <faabric/util/clock.h>

diff --git a/include/faabric/state/State.h b/include/faabric/state/State.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "StateKeyValue.h"
+#include <faabric/state/StateKeyValue.h>
 
 #include <shared_mutex>
 #include <string>

diff --git a/include/faabric/state/StateClient.h b/include/faabric/state/StateClient.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include "InMemoryStateRegistry.h"
-#include "State.h"
+#include <faabric/state/InMemoryStateRegistry.h>
+#include <faabric/state/State.h>
 
 #include <grpcpp/channel.h>
 #include <grpcpp/client_context.h>

diff --git a/include/faabric/state/StateServer.h b/include/faabric/state/StateServer.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "State.h"
+#include <faabric/state/State.h>
 
 #include <faabric/proto/RPCServer.h>
 #include <faabric/proto/faabric.grpc.pb.h>

diff --git a/include/faabric/util/chaining.h b/include/faabric/util/chaining.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include "exception.h"
 #include <faabric/proto/faabric.pb.h>
+#include <faabric/util/exception.h>
 
 namespace faabric::util {
 class ChainedCallFailedException : public faabric::util::FaabricException

diff --git a/include/faabric/util/files.h b/include/faabric/util/files.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include "exception.h"
+#include <faabric/util/exception.h>
 #include <string>
 #include <vector>
 

diff --git a/include/faabric/util/func.h b/include/faabric/util/func.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include "exception.h"
 #include <faabric/proto/faabric.pb.h>
+#include <faabric/util/exception.h>
 #include <string>
 #include <vector>
 

diff --git a/include/faabric/util/http.h b/include/faabric/util/http.h
@@ -1,14 +1,11 @@
 #pragma once
 
-#include "exception.h"
+#include <faabric/util/exception.h>
 #include <pistache/http_header.h>
 #include <string>
 #include <vector>
 
-#define FILE_PATH_HEADER "FilePath"
-#define EMPTY_FILE_RESPONSE "Empty response"
 #define HTTP_FILE_TIMEOUT 20000
-#define IS_DIR_RESPONSE "IS_DIR"
 
 using namespace Pistache;
 
@@ -19,18 +16,10 @@ std::vector<uint8_t> readFileFromUrlWithHeader(
   const std::string& url,
   const std::shared_ptr<Http::Header::Header>& header);
 
-class FileNotFoundAtUrlException : public faabric::util::FaabricException
+class FaabricHttpException : public faabric::util::FaabricException
 {
   public:
-    explicit FileNotFoundAtUrlException(std::string message)
-      : FaabricException(std::move(message))
-    {}
-};
-
-class FileAtUrlIsDirectoryException : public faabric::util::FaabricException
-{
-  public:
-    explicit FileAtUrlIsDirectoryException(std::string message)
+    explicit FaabricHttpException(std::string message)
       : FaabricException(std::move(message))
     {}
 };

diff --git a/include/faabric/util/json.h b/include/faabric/util/json.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include "exception.h"
 #include <faabric/proto/faabric.pb.h>
+#include <faabric/util/exception.h>
 
 namespace faabric::util {
 std::string messageToJson(const faabric::Message& msg);

diff --git a/include/faabric/util/queue.h b/include/faabric/util/queue.h
@@ -1,7 +1,7 @@
 #pragma once
 
-#include "exception.h"
-#include "locks.h"
+#include <faabric/util/exception.h>
+#include <faabric/util/locks.h>
 
 #include <queue>
 

diff --git a/src/endpoint/CMakeLists.txt b/src/endpoint/CMakeLists.txt
@@ -1,5 +1,3 @@
-include_directories(${FAABRIC_INCLUDE_DIR}/faabric/endpoint)
-
 file(GLOB HEADERS "${FAABRIC_INCLUDE_DIR}/faabric/endpoint/*.h")
 
 set(LIB_FILES

diff --git a/src/endpoint/Endpoint.cpp b/src/endpoint/Endpoint.cpp
@@ -1,4 +1,4 @@
-#include "faabric/endpoint/Endpoint.h"
+#include <faabric/endpoint/Endpoint.h>
 
 #include <faabric/util/logging.h>
 #include <pistache/endpoint.h>

diff --git a/src/endpoint/FaabricEndpoint.cpp b/src/endpoint/FaabricEndpoint.cpp
@@ -1,5 +1,5 @@
-#include "FaabricEndpoint.h"
-#include "FaabricEndpointHandler.h"
+#include <faabric/endpoint/FaabricEndpoint.h>
+#include <faabric/endpoint/FaabricEndpointHandler.h>
 
 namespace faabric::endpoint {
 FaabricEndpoint::FaabricEndpoint()

diff --git a/src/endpoint/FaabricEndpointHandler.cpp b/src/endpoint/FaabricEndpointHandler.cpp
@@ -1,4 +1,4 @@
-#include "FaabricEndpointHandler.h"
+#include <faabric/endpoint/FaabricEndpointHandler.h>
 
 #include <faabric/redis/Redis.h>
 #include <faabric/scheduler/Scheduler.h>

diff --git a/src/executor/CMakeLists.txt b/src/executor/CMakeLists.txt
@@ -1,5 +1,3 @@
-include_directories(${FAABRIC_INCLUDE_DIR}/faabric/executor)
-
 file(GLOB HEADERS "${FAABRIC_INCLUDE_DIR}/faabric/executor/*.h")
 
 set(LIB_FILES

diff --git a/src/executor/FaabricExecutor.cpp b/src/executor/FaabricExecutor.cpp
@@ -1,4 +1,4 @@
-#include "FaabricExecutor.h"
+#include <faabric/executor/FaabricExecutor.h>
 
 #include <faabric/state/State.h>
 #include <faabric/util/config.h>

diff --git a/src/executor/FaabricMain.cpp b/src/executor/FaabricMain.cpp
@@ -1,5 +1,4 @@
-#include "FaabricMain.h"
-
+#include <faabric/executor/FaabricMain.h>
 #include <faabric/util/config.h>
 #include <faabric/util/logging.h>
 

diff --git a/src/executor/FaabricPool.cpp b/src/executor/FaabricPool.cpp
@@ -1,6 +1,5 @@
-#include "FaabricPool.h"
-
 #include <faabric/executor/FaabricExecutor.h>
+#include <faabric/executor/FaabricPool.h>
 #include <faabric/util/logging.h>
 
 namespace faabric::executor {