diff --git a/packages/cpp/ArmoniK.Api.Common/CMakeLists.txt b/packages/cpp/ArmoniK.Api.Common/CMakeLists.txt index 2c18ecf9f..79de87970 100644 --- a/packages/cpp/ArmoniK.Api.Common/CMakeLists.txt +++ b/packages/cpp/ArmoniK.Api.Common/CMakeLists.txt @@ -24,12 +24,11 @@ find_package(Threads) include(FetchContent) -FetchContent_Declare( - simdjson - GIT_REPOSITORY https://github.com/simdjson/simdjson.git - GIT_SHALLOW TRUE - GIT_TAG tags/v3.2.1 - ) +FETCHCONTENT_DECLARE( + simdjson + URL https://github.com/simdjson/simdjson/archive/refs/tags/v3.2.1.tar.gz + DOWNLOAD_EXTRACT_TIMESTAMP TRUE +) FetchContent_GetProperties(simdjson) if(NOT simdjson_POPULATED) diff --git a/packages/cpp/ArmoniK.Api.Common/header/options/ComputePlane.h b/packages/cpp/ArmoniK.Api.Common/header/options/ComputePlane.h index 43140e822..a7151ee8c 100644 --- a/packages/cpp/ArmoniK.Api.Common/header/options/ComputePlane.h +++ b/packages/cpp/ArmoniK.Api.Common/header/options/ComputePlane.h @@ -1,5 +1,6 @@ #pragma once #include +#include /** * @brief The armonik namespace contains classes and functions related to the Armonik API. diff --git a/packages/cpp/ArmoniK.Api.Common/header/utils/EnvConfiguration.h b/packages/cpp/ArmoniK.Api.Common/header/utils/EnvConfiguration.h index eb609decc..78be18343 100644 --- a/packages/cpp/ArmoniK.Api.Common/header/utils/EnvConfiguration.h +++ b/packages/cpp/ArmoniK.Api.Common/header/utils/EnvConfiguration.h @@ -1,3 +1,5 @@ +#pragma once + /** * @file EnvConfiguration.h * @brief Header file for the EnvConfiguration class @@ -15,32 +17,8 @@ class EnvConfiguration : public IConfiguration { /** * @brief Default constructor */ - EnvConfiguration() = default; - - /** - * @brief Gets the value of an environment variable - * @param string The name of the environment variable - * @return The value of the environment variable, or an empty string if not found - */ - [[nodiscard]] std::string get(const std::string &string) const override { - std::string value = std::getenv(string.c_str()); - if (!value.empty()) { - return value; - } - throw std::runtime_error("Can't get server address !"); + EnvConfiguration(){ + add_env_configuration(); } - - /** - * @brief Sets the value of an environment variable - * @param string The name of the environment variable - * @param value The value to set - */ - void set(const std::string &string, const std::string &value) override {} - - /** - * @brief Copies the values of another IConfiguration object into this one - * @param other The IConfiguration object to copy from - */ - void set(const IConfiguration &other) override {} }; } // namespace armonik::api::common::utils diff --git a/packages/cpp/ArmoniK.Api.Common/header/utils/IConfiguration.h b/packages/cpp/ArmoniK.Api.Common/header/utils/IConfiguration.h index 7624fcc0a..042643928 100644 --- a/packages/cpp/ArmoniK.Api.Common/header/utils/IConfiguration.h +++ b/packages/cpp/ArmoniK.Api.Common/header/utils/IConfiguration.h @@ -6,8 +6,10 @@ #pragma once #include +#include #include -#include +#include +#include namespace armonik::api::common::options { class ComputePlane; @@ -35,20 +37,26 @@ class IConfiguration { * @param string Key to look up. * @return The value associated with the key, as a string. */ - [[nodiscard]] virtual std::string get(const std::string &string) const = 0; + [[nodiscard]] std::string get(const std::string &string) const; /** * @brief Set the value associated with the given key. * @param string Key to set the value for. * @param value Value to set for the key. */ - virtual void set(const std::string &string, const std::string &value) = 0; + void set(const std::string &string, const std::string &value); /** * @brief Set the values from another IConfiguration object. * @param other IConfiguration object to copy values from. */ - virtual void set(const IConfiguration &other) = 0; + void set(const IConfiguration &other); + + /** + * @brief List defined values of this configuration. + * @note Does not include environment variables + */ + [[nodiscard]] const std::map& list() const; /** * @brief Add JSON configuration from a file. @@ -68,5 +76,13 @@ class IConfiguration { * @return A ComputePlane object representing the current configuration. */ options::ComputePlane get_compute_plane(); + +private: + /** + * @brief Storage for the key-value pairs. + */ + std::map options_; + std::set above_env_keys_; + bool use_environment_ = false; }; } // namespace armonik::api::common::utils diff --git a/packages/cpp/ArmoniK.Api.Common/header/utils/JsonConfiguration.h b/packages/cpp/ArmoniK.Api.Common/header/utils/JsonConfiguration.h index efafdc358..855d383d8 100644 --- a/packages/cpp/ArmoniK.Api.Common/header/utils/JsonConfiguration.h +++ b/packages/cpp/ArmoniK.Api.Common/header/utils/JsonConfiguration.h @@ -12,11 +12,6 @@ namespace armonik::api::common::utils { */ class JsonConfiguration : public IConfiguration { private: - /** - * Parsed values - */ - std::unordered_map values; - JsonConfiguration() = default; public: @@ -26,26 +21,8 @@ class JsonConfiguration : public IConfiguration { */ explicit JsonConfiguration(const std::string &filepath); + static void fromPath(IConfiguration& config, const std::string &filepath); static JsonConfiguration fromString(const std::string &json_string); - - /** - * @brief Get the value associated with the given key. - * @param string Key to look up. - * @return The value associated with the key, as a string. - */ - [[nodiscard]] std::string get(const std::string &string) const override { return values.at(string); } - - /** - * @brief Set the value associated with the given key. - * @param string Key to set the value for. - * @param value Value to set for the key. - */ - void set(const std::string &string, const std::string &value) override { values.insert_or_assign(string, value); } - - /** - * @brief Set the values from another IConfiguration object. - * @param other IConfiguration object to copy values from. - */ - void set(const IConfiguration &other) override {} + static void fromString(IConfiguration& config, const std::string &json_string); }; } // namespace armonik::api::common::utils diff --git a/packages/cpp/ArmoniK.Api.Common/header/utils/RootConfiguration.h b/packages/cpp/ArmoniK.Api.Common/header/utils/RootConfiguration.h deleted file mode 100644 index 4fa72013b..000000000 --- a/packages/cpp/ArmoniK.Api.Common/header/utils/RootConfiguration.h +++ /dev/null @@ -1,51 +0,0 @@ -/** - * @file RootConfiguration.h - * @brief Definition of a root configuration class that inherits from IConfiguration. - */ - -#pragma once -#include "utils/IConfiguration.h" - -namespace armonik::api::common::utils { -/** - * @class RootConfiguration - * @brief Root configuration class that inherits from IConfiguration. - */ -class RootConfiguration : public IConfiguration { -public: - /** - * @brief Default constructor. - */ - RootConfiguration() = default; - - /** - * @brief Get the value associated with the given key. - * @param key Key to look up. - * @return The value associated with the key, as a string. - */ - [[nodiscard]] std::string get(const std::string &key) const override { - auto pair = options_.find(key); - - return (pair != options_.end()) ? (*pair).second : ""; - } - - /** - * @brief Set the value associated with the given key. - * @param key Key to set the value for. - * @param value Value to set for the key. - */ - void set(const std::string &key, const std::string &value) override { options_[key] = value; } - - /** - * @brief Set the values from another IConfiguration object. - * @param other IConfiguration object to copy values from. - */ - void set(const IConfiguration &other) override {} - -protected: - /** - * @brief Storage for the key-value pairs. - */ - std::unordered_map options_; -}; -} // namespace armonik::api::common::utils diff --git a/packages/cpp/ArmoniK.Api.Common/source/utils/IConfiguration.cpp b/packages/cpp/ArmoniK.Api.Common/source/utils/IConfiguration.cpp index c010b2c38..0dcbed51a 100644 --- a/packages/cpp/ArmoniK.Api.Common/source/utils/IConfiguration.cpp +++ b/packages/cpp/ArmoniK.Api.Common/source/utils/IConfiguration.cpp @@ -1,22 +1,49 @@ #include "utils/IConfiguration.h" #include "options/ComputePlane.h" -#include "utils/EnvConfiguration.h" #include "utils/JsonConfiguration.h" namespace armonik::api::common::utils { IConfiguration &IConfiguration::add_json_configuration(const std::string &file_path) { - JsonConfiguration json_configuration(file_path); - + JsonConfiguration::fromPath(*this, file_path); return *this; } IConfiguration &IConfiguration::add_env_configuration() { - EnvConfiguration env_config; + use_environment_ = true; + above_env_keys_.clear(); + return *this; +} +options::ComputePlane IConfiguration::get_compute_plane() { return *this; } -options::ComputePlane IConfiguration::get_compute_plane() { return *this; } +void IConfiguration::set(const IConfiguration &other) { + for(auto&& [key, value] : other.list()){ + set(key, value); + } +} +void IConfiguration::set(const std::string &key, const std::string &value) { + if(use_environment_){ + above_env_keys_.insert(key); + } + options_[key] = value; +} + +std::string IConfiguration::get(const std::string &string) const { + if(use_environment_ && above_env_keys_.find(string) == above_env_keys_.end()){ + char* value = std::getenv(string.c_str()); + if(value != nullptr){ + return value; + } + } + auto position = options_.find(string); + return position == options_.end() ? "" : position->second; +} + +const std::map &IConfiguration::list() const { + return options_; +} } // namespace armonik::api::common::utils diff --git a/packages/cpp/ArmoniK.Api.Common/source/utils/JsonConfiguration.cpp b/packages/cpp/ArmoniK.Api.Common/source/utils/JsonConfiguration.cpp index 3c553e79a..6dee8d645 100644 --- a/packages/cpp/ArmoniK.Api.Common/source/utils/JsonConfiguration.cpp +++ b/packages/cpp/ArmoniK.Api.Common/source/utils/JsonConfiguration.cpp @@ -10,7 +10,7 @@ using namespace simdjson; * @param prefix Prefix for the key * @param element json element */ -void populate(armonik::api::common::utils::JsonConfiguration &config, const std::string &prefix, +void populate(armonik::api::common::utils::IConfiguration &config, const std::string &prefix, const dom::element &element) { switch (element.type()) { case dom::element_type::ARRAY: { @@ -33,14 +33,22 @@ void populate(armonik::api::common::utils::JsonConfiguration &config, const std: } armonik::api::common::utils::JsonConfiguration::JsonConfiguration(const std::string &json_path) { - dom::parser parser; - populate(*this, "", parser.load(json_path)); + fromPath(*this, json_path); } armonik::api::common::utils::JsonConfiguration armonik::api::common::utils::JsonConfiguration::fromString(const std::string &json_string) { JsonConfiguration config; + fromString(config, json_string); + return std::move(config); +} +void armonik::api::common::utils::JsonConfiguration::fromPath(armonik::api::common::utils::IConfiguration &config, + const std::string &filepath) { + dom::parser parser; + populate(config, "", parser.load(filepath)); +} +void armonik::api::common::utils::JsonConfiguration::fromString(armonik::api::common::utils::IConfiguration &config, + const std::string &json_string) { dom::parser parser; populate(config, "", parser.parse(padded_string(json_string))); - return std::move(config); -} \ No newline at end of file +} diff --git a/packages/cpp/ArmoniK.Api.Tests/header/SubmitterCLientTest.h b/packages/cpp/ArmoniK.Api.Tests/header/SubmitterClientTest.h similarity index 100% rename from packages/cpp/ArmoniK.Api.Tests/header/SubmitterCLientTest.h rename to packages/cpp/ArmoniK.Api.Tests/header/SubmitterClientTest.h diff --git a/packages/cpp/ArmoniK.Api.Tests/source/SubmitterCLientTest.cpp b/packages/cpp/ArmoniK.Api.Tests/source/SubmitterClientTest.cpp similarity index 100% rename from packages/cpp/ArmoniK.Api.Tests/source/SubmitterCLientTest.cpp rename to packages/cpp/ArmoniK.Api.Tests/source/SubmitterClientTest.cpp diff --git a/packages/cpp/ArmoniK.Api.Worker.Tests/source/main.cpp b/packages/cpp/ArmoniK.Api.Worker.Tests/source/main.cpp index d810b921a..aeea64bf7 100644 --- a/packages/cpp/ArmoniK.Api.Worker.Tests/source/main.cpp +++ b/packages/cpp/ArmoniK.Api.Worker.Tests/source/main.cpp @@ -7,7 +7,6 @@ #include "grpcpp/support/sync_stream.h" #include "objects.pb.h" -#include "utils/RootConfiguration.h" #include "utils/WorkerServer.h" #include "worker_common.pb.h" #include "worker_service.grpc.pb.h" @@ -29,7 +28,7 @@ using namespace armonik::api::common::utils; int main(int argc, char **argv) { std::cout << "Starting C++ worker..." << std::endl; - std::shared_ptr config = std::make_shared(); + std::shared_ptr config = std::make_shared(); config->set("ComputePlane__WorkerChannel__Address", "/cache/armonik_worker.sock"); config->set("ComputePlane__AgentChannel__Address", "/cache/armonik_agent.sock"); diff --git a/packages/cpp/CMakeLists.txt b/packages/cpp/CMakeLists.txt index 6ec43c04c..1c95e9c45 100644 --- a/packages/cpp/CMakeLists.txt +++ b/packages/cpp/CMakeLists.txt @@ -29,6 +29,7 @@ add_subdirectory(ArmoniK.Api.Client) if (BUILD_TEST OR WIN32) + set(BUILD_DIR "/app/buildtest") add_subdirectory(ArmoniK.Api.Worker.Tests) add_subdirectory(ArmoniK.Api.Tests) endif() diff --git a/packages/cpp/NOTICE.txt b/packages/cpp/NOTICE.txt new file mode 100644 index 000000000..ff9ddedbe --- /dev/null +++ b/packages/cpp/NOTICE.txt @@ -0,0 +1,2 @@ +This project uses simdjson (https://github.com/simdjson/simdjson) from The simdjson authors under Apache 2.0 license (https://www.apache.org/licenses/LICENSE-2.0) + diff --git a/packages/cpp/tools/BuildEnv.Dockerfile b/packages/cpp/tools/BuildEnv.Dockerfile new file mode 100644 index 000000000..626c30cfb --- /dev/null +++ b/packages/cpp/tools/BuildEnv.Dockerfile @@ -0,0 +1,36 @@ +FROM ubuntu:23.04 + +RUN apt-get update && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt-get install -y \ + ssh \ + gcc \ + g++ \ + gdb \ + clang \ + make \ + ninja-build \ + cmake \ + autoconf \ + automake \ + locales-all \ + build-essential \ + libc-ares-dev \ + protobuf-compiler-grpc \ + grpc-proto \ + libgrpc-dev \ + libgrpc++-dev \ + libprotobuf-dev \ + && apt-get clean + +ENV protobuf_BUILD_TESTS=OFF + +RUN ( \ + echo 'LogLevel DEBUG2'; \ + echo 'PermitRootLogin yes'; \ + echo 'PasswordAuthentication yes'; \ + echo 'Subsystem sftp /usr/lib/openssh/sftp-server'; \ + ) > /etc/ssh/sshd_config_test_clion \ + && mkdir -p /run/sshd + +RUN yes password | passwd root + +CMD ["/usr/sbin/sshd", "-D", "-e", "-f", "/etc/ssh/sshd_config_test_clion"] diff --git a/packages/cpp/tools/Dockerfile.ubuntu b/packages/cpp/tools/Dockerfile.ubuntu index 9daf47701..81aae78b5 100644 --- a/packages/cpp/tools/Dockerfile.ubuntu +++ b/packages/cpp/tools/Dockerfile.ubuntu @@ -12,8 +12,7 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt-ge protobuf-compiler-grpc \ grpc-proto \ libgrpc-dev \ - libgrpc++-dev \ - git + libgrpc++-dev # Set environment variables for protobuf ENV protobuf_BUILD_TESTS=OFF @@ -24,7 +23,6 @@ ENV PATH="/app/install/bin:$PATH" # Print the PATH variable RUN echo $PATH -RUN git config --global --add safe.directory '*' # Set the working directory for building protobuf WORKDIR /app/build diff --git a/packages/cpp/tools/restart-build-env.sh b/packages/cpp/tools/restart-build-env.sh new file mode 100644 index 000000000..54f1af08a --- /dev/null +++ b/packages/cpp/tools/restart-build-env.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -x + +IMAGE_NAME="armonik_api_build_env" +IMAGE_TAG="0.1.0" +CONTAINER_NAME="armonik_api_env" + +docker stop "/${CONTAINER_NAME}" + +script_path="$(readlink -f "${BASH_SOURCE:-$0}")" +script_dir="$(dirname "$script_path")" + +working_dir="$script_dir/../" +cd "$working_dir" +working_dir="$(pwd -P)" +cd - + +proto_path="${script_dir}/../../../Protos/V1/" + +# Change to the Protos directory and store its absolute path +cd $proto_path +proto_dir="$(pwd -P)" +cd - + +# Create an install directory and store its absolute path +mkdir -p "${working_dir}/install" +cd "${working_dir}/install" +install_dir="$(pwd -P)" +cd - + +docker build -t "${IMAGE_NAME}:${IMAGE_TAG}" -f BuildEnv.Dockerfile . + +# Change to the working directory +cd "${working_dir}" + +mkdir -p ${working_dir}/build +mkdir -p ${working_dir}/buildtest +mkdir -p ${install_dir} + +REMOTE_BUILD_ADDRESS="${REMOTE_BUILD_ADDRESS:-"127.0.0.1:2223"}" +docker run --rm -d --cap-add sys_ptrace -p"${REMOTE_BUILD_ADDRESS}":22 --name "${CONTAINER_NAME}" -v "${proto_dir}:/app/proto" -v "${working_dir}:/app/source" -v "${install_dir}:/app/install" -v "${working_dir}/build:/app/build" -v "${working_dir}/buildtest:/app/buildtest" "${IMAGE_NAME}:${IMAGE_TAG}" \ No newline at end of file