Add an HTTP server for hosting of ONNX models

.gitignore

@@ -11,6 +11,7 @@ distribute/*
 *.bin
 cmake_build
 .cmake_build
+cmake-build-debug
 gen
 *~
 .vs

BUILD.md

@@ -53,6 +53,10 @@ The complete list of build options can be found by running `./build.sh (or ./bui
 1. For Windows, just add --x86 argument when launching build.bat
 2. For Linux, it must be built out of a x86 os, --x86 argument also needs be specified to build.sh
 
+## Build ONNX Runtime Server on Linux
+
+1. In the ONNX Runtime root folder, run `./build.sh --config RelWithDebInfo --build_server  --use_openmp --parallel`
+
 ## Build/Test Flavors for CI
 
 ### CI Build Environments

cmake/CMakeLists.txt

@@ -70,6 +70,7 @@ option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF)
 option(onnxruntime_USE_TENSORRT "Build with TensorRT support" OFF)
 option(onnxruntime_ENABLE_LTO "Enable link time optimization, which is not stable on older GCCs" OFF)
 option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF)
+option(onnxruntime_BUILD_SERVER "Build ONNX Runtime Server" OFF)
 option(onnxruntime_USE_FULL_PROTOBUF "Use full protobuf" OFF)
 option(onnxruntime_DISABLE_CONTRIB_OPS "Disable contrib ops" OFF)
 option(onnxruntime_USE_EIGEN_THREADPOOL "Use eigen threadpool. Otherwise OpenMP or a homemade one will be used" OFF)
@@ -607,6 +608,10 @@ if (onnxruntime_BUILD_SHARED_LIB)
   include(onnxruntime.cmake)
 endif()
 
+if (onnxruntime_BUILD_SERVER)
+  include(onnxruntime_server.cmake)
+endif()
+
 # some of the tests rely on the shared libs to be
 # built; hence the ordering
 if (onnxruntime_BUILD_UNIT_TESTS)
@@ -633,3 +638,4 @@ if (onnxruntime_BUILD_CSHARP)
 #  set_property(GLOBAL PROPERTY VS_DOTNET_TARGET_FRAMEWORK_VERSION "netstandard2.0")
   include(onnxruntime_csharp.cmake)
 endif()
+

cmake/get_boost.cmake

@@ -0,0 +1,100 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+set(BOOST_REQUESTED_VERSION 1.69.0 CACHE STRING "")
+set(BOOST_SHA1 8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406 CACHE STRING "")
+set(BOOST_USE_STATIC_LIBS true CACHE BOOL "")
+
+set(BOOST_COMPONENTS program_options system thread)
+
+# These components are only needed for Windows
+if(WIN32)
+  list(APPEND BOOST_COMPONENTS date_time regex)
+endif()
+
+# MSVC doesn't set these variables
+if(WIN32)
+  set(CMAKE_STATIC_LIBRARY_PREFIX lib)
+  set(CMAKE_SHARED_LIBRARY_PREFIX lib)
+endif()
+
+# Set lib prefixes and suffixes for linking
+if(BOOST_USE_STATIC_LIBS)
+  set(LIBRARY_PREFIX ${CMAKE_STATIC_LIBRARY_PREFIX})
+  set(LIBRARY_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX})
+else()
+  set(LIBRARY_PREFIX ${CMAKE_SHARED_LIBRARY_PREFIX})
+  set(LIBRARY_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX})
+endif()
+
+# Create list of components in Boost format
+foreach(component ${BOOST_COMPONENTS})
+  list(APPEND BOOST_COMPONENTS_FOR_BUILD --with-${component})
+endforeach()
+
+set(BOOST_ROOT_DIR ${CMAKE_BINARY_DIR}/boost CACHE PATH "")
+
+# TODO: let user give their own Boost installation
+macro(DOWNLOAD_BOOST)
+  if(NOT BOOST_REQUESTED_VERSION)
+    message(FATAL_ERROR "BOOST_REQUESTED_VERSION is not defined.")
+  endif()
+
+  string(REPLACE "." "_" BOOST_REQUESTED_VERSION_UNDERSCORE ${BOOST_REQUESTED_VERSION})
+
+  set(BOOST_MAYBE_STATIC)
+  if(BOOST_USE_STATIC_LIBS)
+    set(BOOST_MAYBE_STATIC "link=static")
+  endif()
+
+  set(VARIANT "release")
+  if(CMAKE_BUILD_TYPE MATCHES Debug)
+    set(VARIANT "debug")
+  endif()
+
+  set(WINDOWS_B2_OPTIONS)
+  set(WINDOWS_LIB_NAME_SCHEME)
+  if(WIN32)
+    set(BOOTSTRAP_FILE_TYPE "bat")
+    set(WINDOWS_B2_OPTIONS "toolset=msvc-14.1" "architecture=x86" "address-model=64")
+    set(WINDOWS_LIB_NAME_SCHEME "-vc141-mt-gd-x64-1_69")
+  else()
+    set(BOOTSTRAP_FILE_TYPE "sh")
+  endif()
+
+  message(STATUS "Adding Boost components")
+  include(ExternalProject)
+  ExternalProject_Add(
+      Boost
+      URL http://dl.bintray.com/boostorg/release/${BOOST_REQUESTED_VERSION}/source/boost_${BOOST_REQUESTED_VERSION_UNDERSCORE}.tar.bz2
+      URL_HASH SHA256=${BOOST_SHA1}
+      DOWNLOAD_DIR ${BOOST_ROOT_DIR}
+      SOURCE_DIR ${BOOST_ROOT_DIR}
+      UPDATE_COMMAND ""
+      CONFIGURE_COMMAND ./bootstrap.${BOOTSTRAP_FILE_TYPE} --prefix=${BOOST_ROOT_DIR}
+      BUILD_COMMAND ./b2 install ${BOOST_MAYBE_STATIC} --prefix=${BOOST_ROOT_DIR} variant=${VARIANT} ${WINDOWS_B2_OPTIONS} ${BOOST_COMPONENTS_FOR_BUILD}
+      BUILD_IN_SOURCE true
+      INSTALL_COMMAND ""
+      INSTALL_DIR ${BOOST_ROOT_DIR}
+  )
+
+  # Set include folders
+  ExternalProject_Get_Property(Boost INSTALL_DIR)
+  set(Boost_INCLUDE_DIR ${INSTALL_DIR}/include)
+  if(WIN32)
+    set(Boost_INCLUDE_DIR ${INSTALL_DIR}/include/boost-1_69)
+  endif()
+
+  # Set libraries to link
+  macro(libraries_to_fullpath varname)
+    set(${varname})
+    foreach(component ${BOOST_COMPONENTS})
+      list(APPEND ${varname} ${INSTALL_DIR}/lib/${LIBRARY_PREFIX}boost_${component}${WINDOWS_LIB_NAME_SCHEME}${LIBRARY_SUFFIX})
+    endforeach()
+  endmacro()
+
+  libraries_to_fullpath(Boost_LIBRARIES)
+  mark_as_advanced(Boost_LIBRARIES Boost_INCLUDE_DIR)
+endmacro()
+
+DOWNLOAD_BOOST()

cmake/onnxruntime_server.cmake

@@ -0,0 +1,122 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+set(SERVER_APP_NAME "onnxruntime_server")
+
+# Generate .h and .cc files from protobuf file
+add_library(server_proto ${ONNXRUNTIME_ROOT}/server/protobuf/predict.proto)
+if(WIN32)
+  target_compile_options(server_proto PRIVATE "/wd4125" "/wd4456")
+endif()
+target_include_directories(server_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES> "${CMAKE_CURRENT_BINARY_DIR}/.." ${CMAKE_CURRENT_BINARY_DIR}/onnx)
+target_compile_definitions(server_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_COMPILE_DEFINITIONS>)
+onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${REPO_ROOT}/cmake/external/protobuf/src ${ONNXRUNTIME_ROOT}/server/protobuf ${ONNXRUNTIME_ROOT}/core/protobuf TARGET server_proto)
+add_dependencies(server_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
+if(NOT WIN32)
+  if(HAS_UNUSED_PARAMETER)
+    set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/model_metadata.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+    set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/model_status.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+    set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/predict.pb.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+  endif()
+endif()
+
+# Setup dependencies
+include(get_boost.cmake)
+set(re2_src ${REPO_ROOT}/cmake/external/re2)
+
+# Setup source code
+set(onnxruntime_server_lib_srcs
+  "${ONNXRUNTIME_ROOT}/server/http/json_handling.cc"
+  "${ONNXRUNTIME_ROOT}/server/http/predict_request_handler.cc"
+  "${ONNXRUNTIME_ROOT}/server/http/util.cc"
+  "${ONNXRUNTIME_ROOT}/server/environment.cc"
+  "${ONNXRUNTIME_ROOT}/server/executor.cc"
+  "${ONNXRUNTIME_ROOT}/server/converter.cc"
+  "${ONNXRUNTIME_ROOT}/server/util.cc"
+  )
+if(NOT WIN32)
+  if(HAS_UNUSED_PARAMETER)
+    set_source_files_properties(${ONNXRUNTIME_ROOT}/server/http/json_handling.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+    set_source_files_properties(${ONNXRUNTIME_ROOT}/server/http/predict_request_handler.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+    set_source_files_properties(${ONNXRUNTIME_ROOT}/server/executor.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+    set_source_files_properties(${ONNXRUNTIME_ROOT}/server/converter.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+    set_source_files_properties(${ONNXRUNTIME_ROOT}/server/util.cc PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+  endif()
+endif()
+
+file(GLOB_RECURSE onnxruntime_server_http_core_lib_srcs
+  "${ONNXRUNTIME_ROOT}/server/http/core/*.cc"
+  )
+
+file(GLOB_RECURSE onnxruntime_server_srcs
+  "${ONNXRUNTIME_ROOT}/server/main.cc"
+)
+
+# HTTP core library
+add_library(onnxruntime_server_http_core_lib STATIC
+  ${onnxruntime_server_http_core_lib_srcs})
+target_include_directories(onnxruntime_server_http_core_lib
+  PUBLIC
+  ${ONNXRUNTIME_ROOT}/server/http/core
+  ${Boost_INCLUDE_DIR}
+  ${re2_src}
+)
+add_dependencies(onnxruntime_server_http_core_lib Boost)
+target_link_libraries(onnxruntime_server_http_core_lib PRIVATE
+  ${Boost_LIBRARIES}
+)
+
+# Server library
+add_library(onnxruntime_server_lib ${onnxruntime_server_lib_srcs})
+onnxruntime_add_include_to_target(onnxruntime_server_lib gsl onnx_proto server_proto)
+target_include_directories(onnxruntime_server_lib PRIVATE
+  ${ONNXRUNTIME_ROOT}
+  ${CMAKE_CURRENT_BINARY_DIR}/onnx
+  ${ONNXRUNTIME_ROOT}/server
+  ${ONNXRUNTIME_ROOT}/server/http
+  PUBLIC
+  ${Boost_INCLUDE_DIR}
+  ${re2_src}
+)
+
+target_link_libraries(onnxruntime_server_lib PRIVATE
+  server_proto
+  ${Boost_LIBRARIES}
+  onnxruntime_server_http_core_lib
+  onnxruntime_session
+  onnxruntime_optimizer
+  onnxruntime_providers
+  onnxruntime_util
+  onnxruntime_framework
+  onnxruntime_util
+  onnxruntime_graph
+  onnxruntime_common
+  onnxruntime_mlas
+  ${onnxruntime_EXTERNAL_LIBRARIES}
+)
+
+# For IDE only
+source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_server_srcs} ${onnxruntime_server_lib_srcs} ${onnxruntime_server_lib})
+
+# Server Application
+add_executable(${SERVER_APP_NAME} ${onnxruntime_server_srcs})
+add_dependencies(${SERVER_APP_NAME} onnx server_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
+
+if(NOT WIN32)
+  if(HAS_UNUSED_PARAMETER)
+    set_source_files_properties("${ONNXRUNTIME_ROOT}/server/main.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+  endif()
+endif()
+
+onnxruntime_add_include_to_target(${SERVER_APP_NAME} onnxruntime_session onnxruntime_server_lib gsl onnx onnx_proto server_proto)
+
+target_include_directories(${SERVER_APP_NAME} PRIVATE
+    ${ONNXRUNTIME_ROOT}
+    ${ONNXRUNTIME_ROOT}/server/http
+)
+
+target_link_libraries(${SERVER_APP_NAME} PRIVATE
+    onnxruntime_server_http_core_lib
+    onnxruntime_server_lib
+)
+

cmake/onnxruntime_unittests.cmake

@@ -162,13 +162,15 @@ set(onnxruntime_test_framework_libs
   onnxruntime_mlas
   )
 
+set(onnxruntime_test_server_libs
+  onnxruntime_test_utils
+  onnxruntime_test_utils_for_server
+)
 
 if(WIN32)
     list(APPEND onnxruntime_test_framework_libs Advapi32)
 endif()
 
-
-
 set (onnxruntime_test_providers_dependencies ${onnxruntime_EXTERNAL_DEPENDENCIES})
 
 if(onnxruntime_USE_CUDA)
@@ -550,6 +552,58 @@ if (onnxruntime_BUILD_SHARED_LIB)
   endif()
 endif()
 
+if (onnxruntime_BUILD_SERVER)
+  file(GLOB onnxruntime_test_server_src
+    "${TEST_SRC_DIR}/server/unit_tests/*.cc"
+    "${TEST_SRC_DIR}/server/unit_tests/*.h"
+  )
+
+  file(GLOB onnxruntime_integration_test_server_src
+    "${TEST_SRC_DIR}/server/integration_tests/*.py"
+  )
+  if(NOT WIN32)
+    if(HAS_UNUSED_PARAMETER)
+      set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/json_handling_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+      set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/converter_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+      set_source_files_properties("${TEST_SRC_DIR}/server/unit_tests/util_tests.cc" PROPERTIES COMPILE_FLAGS -Wno-unused-parameter)
+    endif()
+  endif()
+
+  add_library(onnxruntime_test_utils_for_server ${onnxruntime_test_server_src})
+  onnxruntime_add_include_to_target(onnxruntime_test_utils_for_server onnxruntime_test_utils gtest gmock gsl onnx onnx_proto server_proto)
+  add_dependencies(onnxruntime_test_utils_for_server onnxruntime_server_lib onnxruntime_server_http_core_lib Boost ${onnxruntime_EXTERNAL_DEPENDENCIES})
+  target_include_directories(onnxruntime_test_utils_for_server PUBLIC ${Boost_INCLUDE_DIR} ${REPO_ROOT}/cmake/external/re2 ${CMAKE_CURRENT_BINARY_DIR}/onnx ${ONNXRUNTIME_ROOT}/server/http ${ONNXRUNTIME_ROOT}/server/http/core PRIVATE ${ONNXRUNTIME_ROOT} )
+  target_link_libraries(onnxruntime_test_utils_for_server ${Boost_LIBRARIES} ${onnx_test_libs})
+
+  AddTest(
+    TARGET onnxruntime_server_tests
+    SOURCES ${onnxruntime_test_server_src}
+    LIBS ${onnxruntime_test_server_libs} server_proto onnxruntime_server_lib ${onnxruntime_test_providers_libs}
+    DEPENDS ${onnxruntime_EXTERNAL_DEPENDENCIES}
+  )
+
+  onnxruntime_protobuf_generate(
+          APPEND_PATH IMPORT_DIRS ${REPO_ROOT}/cmake/external/protobuf/src ${ONNXRUNTIME_ROOT}/server/protobuf ${ONNXRUNTIME_ROOT}/core/protobuf
+          PROTOS ${ONNXRUNTIME_ROOT}/server/protobuf/predict.proto ${ONNXRUNTIME_ROOT}/server/protobuf/onnx-ml.proto
+          LANGUAGE python
+          TARGET onnxruntime_server_tests
+          OUT_VAR server_test_py)
+
+  add_custom_command(
+    TARGET onnxruntime_server_tests POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/server_test
+    COMMAND ${CMAKE_COMMAND} -E copy
+      ${onnxruntime_integration_test_server_src}
+      ${CMAKE_CURRENT_BINARY_DIR}/server_test/
+    COMMAND ${CMAKE_COMMAND} -E copy
+      ${CMAKE_CURRENT_BINARY_DIR}/onnx_ml_pb2.py
+      ${CMAKE_CURRENT_BINARY_DIR}/server_test/
+    COMMAND ${CMAKE_COMMAND} -E copy
+      ${CMAKE_CURRENT_BINARY_DIR}/predict_pb2.py
+      ${CMAKE_CURRENT_BINARY_DIR}/server_test/
+  )
+
+endif()
 
 add_executable(onnxruntime_mlas_test ${TEST_SRC_DIR}/mlas/unittest.cpp)
 target_include_directories(onnxruntime_mlas_test PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc)