Merge pull request #256 from deephealthproject/develop

Develop
deephealthproject · Feb 12, 2021 · 8fd26ff · 8fd26ff
2 parents df82c9e + 90d016b
commit 8fd26ff
Show file tree

Hide file tree

Showing 102 changed files with 15,538 additions and 877 deletions.
diff --git a/.github/workflows/build-cpu-windows.yml b/.github/workflows/build-cpu-windows.yml
@@ -21,7 +21,7 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_SUPERBUILD=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_TARGET=CPU -DBUILD_HPC=OFF -DBUILD_TESTS=ON ..
+        cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_SUPERBUILD=ON -DBUILD_SHARED_LIBS=OFF -DBUILD_TARGET=CPU -DBUILD_HPC=OFF -DBUILD_TESTS=ON -DBUILD_DIST=OFF -DBUILD_RUNTIME=OFF ..
       shell: cmd
     - name: Build
       run: cmake --build build --config Release

diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml
@@ -16,12 +16,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Install utilities
-        run: sudo apt-get install -y cmake wget graphviz
+        run: |
+          sudo apt-get install -y cmake wget graphviz
+          #sudo apt-get install -y build-essential checkinstall zlib1g-dev libcrypto++-dev libssl-dev
       - name: Build
         run: |
           mkdir build
           cd build
-          cmake .. -DBUILD_SUPERBUILD=ON -DBUILD_HPC=OFF -DBUILD_TESTS=ON
+          cmake .. -DBUILD_SUPERBUILD=ON -DBUILD_HPC=OFF -DBUILD_TESTS=ON -DBUILD_DIST=OFF -DBUILD_RUNTIME=OFF
           make
       - name: Test
         run: |

diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,7 @@ src/serialization/onnx/onnx.pb.cc
 *.DS_Store
 .idea
 .vscode
+*~
 
 # Build
 /[Bb]uild*

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,15 +9,18 @@ cmake_minimum_required(VERSION 3.9.2)
 option(BUILD_SUPERBUILD "Compile using the superbuild system" OFF)
 option(BUILD_PROTOBUF "Compile using Protobuf" ON)
 option(BUILD_OPENMP "Compile using OpenMP" ON)
-option(BUILD_HPC "Compile using aggressive flags" ON)
+option(BUILD_HPC "Compile using aggressive flags for performance" ON)
 option(BUILD_TESTS "Compile tests (HPC needs to be disabled)" OFF)  # Disable HPC to pass tests (there are numerical errors)
 option(BUILD_EXAMPLES "Compile examples" ON)
-option(BUILD_SHARED_LIBS "Global flag to cause add_library to create shared libraries if on" ON)
+option(BUILD_DIST "Compile for a distributed execution" OFF)
+option(BUILD_RUNTIME "Compile runtime" OFF)
 option(BUILD_COVERAGE "Flag to compile for coverage information" OFF)
 option(BUILD_SANITIZERS "Flag to compile with sanitizers information" OFF)
 
 if(WIN32)
-    option(BUILD_SHARED_LIBS "" OFF) # Prefer lib over dll in windows
+    option(BUILD_SHARED_LIBS "Global flag to cause add_library to create shared libraries if on" OFF)
+else()
+    option(BUILD_SHARED_LIBS "Global flag to cause add_library to create shared libraries if on" ON)
 endif()
 
 ###########################################################################
@@ -144,6 +147,11 @@ if(BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif(BUILD_EXAMPLES)
 
+# Build runtime
+if(BUILD_RUNTIME)
+    add_subdirectory(runtime)
+endif(BUILD_RUNTIME)
+
 
 ###########################################################################
 ########################## INSTALLATION ###################################

diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -20,11 +20,12 @@ endif()
 # For development (ignore)
 option(USE_SYSTEM_GTEST "Use system dependency" OFF)
 option(USE_SYSTEM_EIGEN "Use system dependency" OFF)
-option(USE_SYSTEM_ZLIB "Use system dependency" OFF)
 option(USE_SYSTEM_PROTOBUF "Use system dependency" OFF)
-if(WIN32)
-    option(USE_SYSTEM_PTHREADS "Use system dependency" OFF)
-endif()
+option(USE_SYSTEM_ZLIB "Use system dependency" OFF)
+option(USE_SYSTEM_OPENSSL "Use system dependency" OFF)
+#if(WIN32)
+#    option(USE_SYSTEM_PTHREADS "Use system dependency" OFF)
+#endif()
 
 # Set variables
 set(EDDL_DEPENDENCIES)
@@ -73,14 +74,14 @@ if(CMAKE_GENERATOR MATCHES "Visual Studio")
     list(APPEND config_types ${CMAKE_BUILD_TYPE})
 endif()
 
-##############
-###  GTEST  ##
-##############
+#############
+##  GTEST  ##
+#############
 message(STATUS "Subproject: GTEST...")
 if(USE_SYSTEM_GTEST)
     find_package(GTest REQUIRED)
 else()
-    # Download and unpack googletest at configure time   
+    # Download and unpack googletest at configure time
     foreach(config ${config_types})
         CONF_PACKAGE(googletest ${config} "")
         BUILD_PACKAGE(googletest ${config})
@@ -120,31 +121,74 @@ else()
         CONF_PACKAGE(protobuf ${config} @ONLY)
         BUILD_PACKAGE(protobuf ${config})
     endforeach()
-    
+
     set(Protobuf_ROOT "${EP_BASE_DIR}/protobuf" PARENT_SCOPE)
     set(Protobuf_INCLUDE_DIRS "${EP_BASE_DIR}/protobuf/include" PARENT_SCOPE)
     set(Protobuf_PROTOC_EXECUTABLE "${EP_BASE_DIR}/protobuf/bin/protoc" PARENT_SCOPE)
 endif()
-add_custom_target(protobuf_files 
+add_custom_target(protobuf_files
 	protoc --cpp_out=../src/serialization/onnx ../src/serialization/onnx/onnx.proto
 	)
 message(STATUS "Subproject: Protobuf...DONE")
 
-##################
-##  PTHREADS4W  ##
-##################
-if(WIN32)
-    if(USE_SYSTEM_PTHREADS)
-        set(PTHREADS_INSTALL_PATH "$ENV{PTHREADS_ROOT}" CACHE PATH "Path to the installation of Pthreads under Windows (PTHREADS_ROOT env variable)" PARENT_SCOPE)
+
+# Only for distributed versions
+if(BUILD_DIST)
+
+    ############
+    ##  ZLIB  ##
+    ############
+    message(STATUS "Subproject: ZLIB...")
+    if(USE_SYSTEM_ZLIB)
+        find_package(ZLIB REQUIRED)
     else()
-        # Download and unpack pthreads4w at configure time
-        CONF_PACKAGE(pthreads4w ${CMAKE_BUILD_TYPE} @ONLY)
-        BUILD_PACKAGE(pthreads4w ${CMAKE_BUILD_TYPE})
-
-        set(PTHREADS_INSTALL_PATH "${EP_BASE_DIR}/PTHREADS-BUILT" PARENT_SCOPE)
+        # Download and unpack ZLIB at configure time
+        foreach(config ${config_types})
+            CONF_PACKAGE(zlib ${config} "")
+            BUILD_PACKAGE(zlib ${config})
+        endforeach()
+        # Set variables
+        SET(ZLIB_ROOT "${EP_BASE_DIR}/zlib" PARENT_SCOPE)
+        SET(ZLIB_INCLUDE_DIRS "${EP_BASE_DIR}/zlib/include" PARENT_SCOPE)
     endif()
+    message(STATUS "Subproject: ZLIB...DONE")
+
+
+    ###############
+    ##  OPENSSL  ##
+    ###############
+    message(STATUS "Subproject: OpenSSL...")
+    if(USE_SYSTEM_OPENSSL)
+        find_package(OpenSSL REQUIRED)
+    else()
+        # Download and unpack ZLIB at configure time
+        foreach(config ${config_types})
+            CONF_PACKAGE(openssl ${config} "")
+            BUILD_PACKAGE(openssl ${config})
+        endforeach()
+        # Set variables
+        SET(OPENSSL_ROOT_DIR "${EP_BASE_DIR}/openssl" PARENT_SCOPE)
+        SET(OPENSSL_INCLUDE_DIR "${EP_BASE_DIR}/openssl/include" PARENT_SCOPE)
+    endif()
+    message(STATUS "Subproject: OpenSSL...DONE")
+
 endif()
 
+##################
+##  PTHREADS4W  ##
+##################
+#if(WIN32)
+#    if(USE_SYSTEM_PTHREADS)
+#        set(PTHREADS_INSTALL_PATH "$ENV{PTHREADS_ROOT}" CACHE PATH "Path to the installation of Pthreads under Windows (PTHREADS_ROOT env variable)" PARENT_SCOPE)
+#    else()
+#        # Download and unpack pthreads4w at configure time
+#        CONF_PACKAGE(pthreads4w ${CMAKE_BUILD_TYPE} @ONLY)
+#        BUILD_PACKAGE(pthreads4w ${CMAKE_BUILD_TYPE})
+#
+#        set(PTHREADS_INSTALL_PATH "${EP_BASE_DIR}/PTHREADS-BUILT" PARENT_SCOPE)
+#    endif()
+#endif()
+
 ###############
 #####  EDDL  ##
 ###############

diff --git a/cmake/EDDLConfig.cmake.in b/cmake/EDDLConfig.cmake.in
@@ -3,14 +3,19 @@ include(CMakeFindDependencyMacro)
 
 # VARIABLES
 SET(USE_SUPERBUILD @BUILD_SUPERBUILD@)  # Set in parent
+SET(USE_DIST @BUILD_DIST@)  # Set in parent
+
 SET(USE_PROTOBUF @BUILD_PROTOBUF@)  # Set in parent
 SET(USE_OPENMP @USE_OPENMP@)  # Modified in a subdirectory
+SET(USE_DIST @USE_DIST@)  # Modified in a subdirectory
 SET(USE_CUDA @USE_CUDA@) # Modified in a subdirectory
 SET(USE_CUDNN @USE_CUDNN@) # Modified in a subdirectory
 SET(USE_FPGA @USE_FPGA@) # Modified in a subdirectory
+
 SET(USE_SYSTEM_EIGEN @USE_SYSTEM_EIGEN@) # Modified in a subdirectory
-#SET(USE_SYSTEM_ZLIB @USE_SYSTEM_ZLIB@) # Modified in a subdirectory
 SET(USE_SYSTEM_PROTOBUF @USE_SYSTEM_PROTOBUF@) # Modified in a subdirectory
+SET(USE_SYSTEM_ZLIB ON) # Modified in a subdirectory  (the superbuild installation does not include the cmakes to find the library)
+SET(USE_SYSTEM_OPENSSL ON) # Modified in a subdirectory (the superbuild installation does not include the cmakes to find the library)
 SET(EP_BASE_DIR @EP_BASE_DIR@) # Modified in a subdirectory
 
 # Threads (linux)
@@ -35,6 +40,11 @@ if(USE_FPGA)
     find_dependency(OpenCL REQUIRED)
 endif()
 
+# Distributed
+if(USE_DIST)
+    find_dependency(OpenSSL REQUIRED)
+endif()
+
 if(USE_SUPERBUILD)
     list(APPEND CMAKE_MODULE_PATH "${EP_BASE_DIR}")
     if(NOT USE_SYSTEM_EIGEN)
@@ -56,6 +66,24 @@ if(USE_PROTOBUF)
     endif()
 endif()
 
+if(USE_DIST)
+    # Zlib
+    if(USE_SUPERBUILD AND NOT USE_SYSTEM_ZLIB)
+        SET(ZLIB_ROOT @ZLIB_ROOT@) # Modified in a subdirectory
+        find_dependency(ZLIB CONFIG REQUIRED HINTS ${ZLIB_ROOT})
+    else()
+        find_dependency(ZLIB REQUIRED)
+    endif()
+
+    # OpenSSL
+    if(USE_SUPERBUILD AND NOT USE_SYSTEM_OPENSSL)
+        SET(OPENSSL_ROOT_DIR @OPENSSL_ROOT_DIR@) # Modified in a subdirectory
+        find_dependency(OpenSSL CONFIG REQUIRED HINTS ${OPENSSL_ROOT_DIR})
+    else()
+        find_dependency(OpenSSL REQUIRED)
+    endif()
+endif()
+
 include("${CMAKE_CURRENT_LIST_DIR}/EDDLTargets.cmake")
 
 # Set default paths

diff --git a/cmake/openssl.CMakeLists.txt.in b/cmake/openssl.CMakeLists.txt.in
@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.9.2)
+
+project(openssl-download NONE)
+
+# Set base dir
+SET(EP_BASE_DIR @EP_BASE_DIR@)
+SET(CMAKE_BUILD_TYPE @CMAKE_BUILD_TYPE@)
+
+include(ExternalProject)
+ExternalProject_Add(openssl
+		PREFIX openssl
+		GIT_REPOSITORY "https://github.com/openssl/openssl.git"
+		GIT_TAG "OpenSSL_1_1_1i"
+		SOURCE_DIR "${EP_BASE_DIR}/openssl-src"
+		BINARY_DIR "${EP_BASE_DIR}/openssl-build"
+        INSTALL_DIR "${EP_BASE_DIR}/openssl"
+		CONFIGURE_COMMAND sh ${EP_BASE_DIR}/openssl-src/config --prefix=${EP_BASE_DIR}/openssl
+		)
diff --git a/docs/markdown/bundle/requirements.txt b/docs/markdown/bundle/requirements.txt
@@ -13,10 +13,7 @@ ca-certificates           2020.12.5            ha878542_0    conda-forge
 cairo                     1.16.0            h7979940_1007    conda-forge
 certifi                   2020.12.5        py38h578d9bd_1    conda-forge
 chardet                   4.0.0                    pypi_0    pypi
-cmake                     3.19.4               h4547794_0    conda-forge
-cudatoolkit               11.0.3               h15472ef_7    conda-forge
-cudatoolkit-dev           10.1.243             h516909a_3    conda-forge
-cudnn                     8.0.5.39             ha5ca753_1    conda-forge
+cmake                     3.19.4               h3020d66_1    conda-forge
 docutils                  0.16                     pypi_0    pypi
 doxygen                   1.9.1                hb166930_0    conda-forge
 eigen                     3.3.7             hc9558a2_1001    conda-forge
@@ -51,7 +48,7 @@ libedit                   3.1.20191231         he28a2e2_2    conda-forge
 libev                     4.33                 h516909a_1    conda-forge
 libffi                    3.3                  h58526e2_2    conda-forge
 libgcc-ng                 9.3.0               h2828fa1_18    conda-forge
-libglib                   2.66.4               h748fe8e_2    conda-forge
+libglib                   2.66.6               h1f3bc88_3    conda-forge
 libgomp                   9.3.0               h2828fa1_18    conda-forge
 libiconv                  1.16                 h516909a_0    conda-forge
 libidn2                   2.3.0                h516909a_0    conda-forge

diff --git a/docs/sphinx/source/intro/build-options.rst b/docs/sphinx/source/intro/build-options.rst
@@ -25,31 +25,31 @@ environment by running the following commands **from the source directory**:
 
 .. code:: bash
 
-    conda env create -f environment-cpu.yml  # -cpu, -gpu, -cudnn
+    conda env create -f environment.yml
     conda activate eddl
 
 You can also update your environment with:
 
 .. code:: bash
 
-    conda env update -f environment-cpu.yml  # -cpu, -gpu, -cudnn
+    conda env update -f environment.yml
 
 If you decide to manually install these dependencies in your system (make sure they are at standard paths):
 
 .. code:: yaml
 
-    - cmake>=3.9.2
+    - cmake>=3.17.2
     - eigen==3.3.7
     - protobuf==3.11.4
-    - libprotobuf==3.11.4
-    - cudnn==8.0.5.39
-    - cudatoolkit-dev==10.1.243
-    - gtest
-    - graphviz
-    - wget
-    - doxygen
-    - python
-    - pip
+    - libprotobuf==3.11.4  # We need to avoid problems with paths (idk why)
+    - zlib==1.2.11
+    - openssl==1.1.1i
+    - gtest==1.10.0
+    - graphviz==2.42.3  # Build & Run
+    - wget==1.20.1
+    - doxygen==1.9.1  # Docs
+    - python==3.8.6
+    - pip==21.0.1
     - pip:
         - sphinx==3.2.1
         - sphinx_rtd_theme==0.5.0
@@ -134,7 +134,7 @@ not found (or CUDA), the EDDL will automatically fallback to CPU.
 Additional flags
 ^^^^^^^^^^^^^^^^^
 
-These flags can enable/disable features of the EDDL so that you can optimized and
+These flags can enable/disable features of the EDDL so that you can optimize and
 troubleshoot the compilation process (see: :doc:`troubleshoot`).
 
 
@@ -181,8 +181,18 @@ troubleshoot the compilation process (see: :doc:`troubleshoot`).
 
 .. note::
 
-    This flag is needed to known which CUDA Toolkit/cuDNN the user wants to use. By default cmake looks in the ``PATH``.
+    This flag is needed to known which CUDA Toolkit the user wants to use. By default cmake looks in the ``PATH``.
 
+- **CUDNN ROOT DIR:**
+
+.. code:: bash
+
+    --DCUDNN_ROOT_DIR=/path/to/cuda  #/usr/local/cuda
+
+.. note::
+
+    This flag is needed to known where to look for the cuDNN libraries. By default cuda is expected to be installed in
+    along with the CUDA toolkit.
 
 - **CUDA host compiler:**
 
@@ -296,8 +306,17 @@ troubleshoot the compilation process (see: :doc:`troubleshoot`).
     If you want to distribute the resulting shared library, you should use the flag
     ``-DBUILD_SUPERBUILD=ON`` so that we can make specific tunings to our dependencies.
 
+- **Build distributed:** To let the EDDL work in a distributed mode, use the setting ``BUILD_DIST``:
+
+.. code:: bash
+
+    -DBUILD_DIST=ON
+
+.. note::
+
+    Enabled by default.
 
 
 .. _Anaconda: https://docs.conda.io/en/latest/miniconda.html
 .. _Eigen3: http://eigen.tuxfamily.org/index.php?title=Main_Page
-.. _Requirements: https://github.com/deephealthproject/eddl/blob/develop/docs/markdown/bundle/requirements.txt
+.. _Requirements: https://github.com/deephealthproject/eddl/blob/develop/docs/markdown/bundle/requirements.txt
-Original file line number
+Diff line change
@@ Expand Up / @@ -10,6 +10,7 @@ src/serialization/onnx/onnx.pb.cc @@
     *.DS_Store
     .idea
     .vscode
+    *~
     # Build
     /[Bb]uild*
@@ Expand Down @@