Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Shared object build #36

Merged
merged 13 commits into from
Dec 8, 2020
3 changes: 2 additions & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on:
branches: [master]
pull_request:
branches: [master]
types: [opened, synchronize, reopened, ready_for_review]

jobs:
tests:
Expand All @@ -15,7 +16,7 @@ jobs:
REDIS_QUEUE_HOST: redis
REDIS_STATE_HOST: redis
container:
image: faasm/faabric-cli:0.0.13
image: faasm/faabric-cli:0.0.14
defaults:
run:
working-directory: /code/faabric
Expand Down
9 changes: 4 additions & 5 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.13.0)
project(faabric)

option(FAABRIC_WASM_BUILD "Build Faabric wasm library" OFF)
option(FAABRIC_STATIC_LIBS "Statically link Faabric libs" ON)
option(FAABRIC_BUILD_TESTS "Build Faabric tests" ON)

# Top-level CMake config
Expand All @@ -18,20 +17,20 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(THIRD_PARTY_INSTALL_DIR ${CMAKE_BINARY_DIR}/third-party)

# Library funcs
if (FAABRIC_STATIC_LIBS)
if (BUILD_SHARED_LIBS)
function(faabric_public_lib lib_name lib_deps)
add_library(${lib_name} STATIC ${lib_deps})
add_library(${lib_name} SHARED ${lib_deps})
endfunction()
function(faabric_private_lib lib_name lib_deps)
add_library(${lib_name} STATIC ${lib_deps})
target_compile_options(${lib_name} PRIVATE "-fPIC")
endfunction()
else ()
function(faabric_public_lib lib_name lib_deps)
add_library(${lib_name} SHARED ${lib_deps})
add_library(${lib_name} STATIC ${lib_deps})
endfunction()
function(faabric_private_lib lib_name lib_deps)
add_library(${lib_name} STATIC ${lib_deps})
target_compile_options(${lib_name} PRIVATE "-fPIC")
endfunction()
endif ()

Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.0.13
0.0.14
6 changes: 6 additions & 0 deletions cmake/ExternalProjects.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ include_directories(${THIRD_PARTY_INSTALL_DIR}/include)
# Protobuf/ grpc config
# See the example in the gRPC repo here:
# https://github.com/grpc/grpc/blob/master/examples/cpp/helloworld/CMakeLists.txt
if(BUILD_SHARED_LIBS)
set(Protobuf_USE_STATIC_LIBS OFF)
else()
set(Protobuf_USE_STATIC_LIBS ON)
endif()

include(FindProtobuf)
set(protobuf_MODULE_COMPATIBLE TRUE)
find_package(Protobuf REQUIRED)
Expand Down
2 changes: 1 addition & 1 deletion docker/faabric.dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM faasm/grpc-root:0.0.12
FROM faasm/grpc-root:0.0.14
ARG FAABRIC_VERSION

# Note - the version of grpc-root here can be quite behind as it's rebuilt very
Expand Down
15 changes: 13 additions & 2 deletions docker/grpc-root.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ RUN apt install -y \
libhiredis-dev \
libtool \
libboost-filesystem-dev \
libcurl4-openssl-dev \
ninja-build \
python3-dev \
python3-pip \
Expand All @@ -32,10 +31,22 @@ RUN wget -q -O \
RUN sh cmake-linux.sh -- --skip-license --prefix=/usr/local

# gRPC, protobuf etc.
# Static libs
RUN git clone --recurse-submodules -b v1.31.0 https://github.com/grpc/grpc
WORKDIR /setup/grpc/cmake/build
WORKDIR /setup/grpc/cmake/build-static
RUN cmake -GNinja \
-DgRPC_INSTALL=ON \
-DBUILD_SHARED_LIBS=OFF \
-DgRPC_BUILD_TESTS=OFF \
-DCMAKE_INSTALL_PREFIX=/usr/local \
../..
RUN ninja install

# Shared libs
WORKDIR /setup/grpc/cmake/build-shared
RUN cmake -GNinja \
-DgRPC_INSTALL=ON \
-DBUILD_SHARED_LIBS=ON \
-DgRPC_BUILD_TESTS=OFF \
-DCMAKE_INSTALL_PREFIX=/usr/local \
../..
Expand Down
4 changes: 0 additions & 4 deletions include/faabric/util/chaining.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@
#include <proto/faabric.pb.h>

namespace faabric::util {
std::string postJsonFunctionCall(const std::string& host,
csegarragonz marked this conversation as resolved.
Show resolved Hide resolved
int port,
const faabric::Message& msg);

class ChainedCallFailedException : public faabric::util::FaabricException
{
public:
Expand Down
24 changes: 0 additions & 24 deletions include/faabric/util/files.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@
#include <string>
#include <vector>

#define FILE_PATH_HEADER "FilePath"
#define EMPTY_FILE_RESPONSE "Empty response"
csegarragonz marked this conversation as resolved.
Show resolved Hide resolved

namespace faabric::util {
std::string readFileToString(const std::string& path);

Expand All @@ -15,26 +12,5 @@ std::vector<uint8_t> readFileToBytes(const std::string& path);
void writeBytesToFile(const std::string& path,
const std::vector<uint8_t>& data);

std::vector<uint8_t> readFileFromUrl(const std::string& url);

std::vector<uint8_t> readFileFromUrlWithHeader(const std::string& url,
const std::string& header);

bool isWasm(const std::vector<uint8_t>& bytes);

class FileNotFoundAtUrlException : public faabric::util::FaabricException
{
public:
explicit FileNotFoundAtUrlException(std::string message)
: FaabricException(std::move(message))
{}
};

class FileAtUrlIsDirectoryException : public faabric::util::FaabricException
{
public:
explicit FileAtUrlIsDirectoryException(std::string message)
: FaabricException(std::move(message))
{}
};
}
37 changes: 37 additions & 0 deletions include/faabric/util/http.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#pragma once

#include "exception.h"
#include <pistache/http_header.h>
#include <string>
#include <vector>

#define FILE_PATH_HEADER "FilePath"
#define EMPTY_FILE_RESPONSE "Empty response"
#define HTTP_FILE_TIMEOUT 20000
#define IS_DIR_RESPONSE "IS_DIR"

using namespace Pistache;

namespace faabric::util {
std::vector<uint8_t> readFileFromUrl(const std::string& url);

std::vector<uint8_t> readFileFromUrlWithHeader(
const std::string& url,
const std::shared_ptr<Http::Header::Header>& header);

class FileNotFoundAtUrlException : public faabric::util::FaabricException
{
public:
explicit FileNotFoundAtUrlException(std::string message)
: FaabricException(std::move(message))
{}
};

class FileAtUrlIsDirectoryException : public faabric::util::FaabricException
{
public:
explicit FileAtUrlIsDirectoryException(std::string message)
: FaabricException(std::move(message))
{}
};
}
6 changes: 3 additions & 3 deletions src/proto/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ set(LIB_FILES
${OUTPUT_FILES}
)

if(FAABRIC_STATIC_LIBS)
faabric_private_lib(proto "${LIB_FILES}")
else()
if(BUILD_SHARED_LIBS)
faabric_public_lib(proto "${LIB_FILES}")
else()
faabric_private_lib(proto "${LIB_FILES}")
endif()

add_dependencies(proto spdlog_ext)
Expand Down
5 changes: 1 addition & 4 deletions src/util/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ file(GLOB HEADERS "${FAABRIC_INCLUDE_DIR}/faabric/util/*.h")
set(LIB_FILES
barrier.cpp
bytes.cpp
chaining.cpp
config.cpp
clock.cpp
environment.cpp
files.cpp
func.cpp
gids.cpp
http.cpp
json.cpp
logging.cpp
memory.cpp
Expand All @@ -34,9 +34,6 @@ add_dependencies(util rapidjson_ext)
target_link_libraries(
util
proto
ssl
curl
crypto
csegarragonz marked this conversation as resolved.
Show resolved Hide resolved
boost_system
boost_filesystem
)
78 changes: 0 additions & 78 deletions src/util/chaining.cpp

This file was deleted.

66 changes: 3 additions & 63 deletions src/util/files.cpp
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
#include "files.h"
#include "bytes.h"
#include "logging.h"

#include <curl/curl.h>
#include <curl/easy.h>

#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <sstream>

#define HTTP_FILE_TIMEOUT 20000
#include <string>

namespace faabric::util {
std::string readFileToString(const std::string& path)
Expand Down Expand Up @@ -67,64 +65,6 @@ size_t writeDataCallback(void* ptr, size_t size, size_t nmemb, void* stream)
return size * nmemb;
}

std::vector<uint8_t> readFileFromUrl(const std::string& url)
{
return readFileFromUrlWithHeader(url, "");
}

std::vector<uint8_t> readFileFromUrlWithHeader(const std::string& url,
const std::string& header)
{

void* curl = curl_easy_init();

std::stringstream out;
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeDataCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, HTTP_FILE_TIMEOUT);

// Add header
if (!header.empty()) {
struct curl_slist* chunk = nullptr;
chunk = curl_slist_append(chunk, header.c_str());
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
}

// Make the request
CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);

// Check response is OK
if (res != CURLE_OK) {
std::string msg =
std::string("Unable to get file due to curl error ") + url;
throw FileNotFoundAtUrlException(msg);
}

// Check response code
long http_code = 0;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code > 200) {
std::string msg = "Unable to get file " + url +
" response code: " + std::to_string(http_code);
throw FileNotFoundAtUrlException(msg);
}

// Check there's something in the response
if (out.str().empty()) {
std::string msg = "Empty response for file " + url;
throw FileNotFoundAtUrlException(msg);
}

if (out.str() == "IS_DIR") {
throw faabric::util::FileAtUrlIsDirectoryException(url +
" is a directory");
}

return faabric::util::stringToBytes(out.str());
}
csegarragonz marked this conversation as resolved.
Show resolved Hide resolved

bool isWasm(const std::vector<uint8_t>& bytes)
{
static const uint8_t wasmMagicNumber[4] = { 0x00, 0x61, 0x73, 0x6d };
Expand Down
Loading