Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

link mpi when either use_mpi or use_nccl enabled #14467

Merged
merged 5 commits into from
Feb 3, 2023
Merged

link mpi when either use_mpi or use_nccl enabled #14467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a7d7d78
only link mpi when needed
pengwa Jan 29, 2023
169684f
make mpi not found fatal error when it is really needed
pengwa Jan 31, 2023
0c0776d
disable MPI and NCCL by default
pengwa Jan 31, 2023
f6af9ce
fix format
pengwa Feb 2, 2023
14937a7
Merge branch 'main' of https://github.com/microsoft/onnxruntime into …
pengwa Feb 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions cmake/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1366,19 +1366,22 @@ if (onnxruntime_ENABLE_TRAINING)

find_package(MPI)

if (MPI_CXX_FOUND)
message( STATUS "MPI Version: ${MPI_CXX_VERSION}")
message( STATUS "MPI (include: ${MPI_CXX_INCLUDE_DIRS}, library: ${MPI_CXX_LIBRARIES})" )
mark_as_advanced(MPI_CXX_INCLUDE_DIRS MPI_CXX_LIBRARIES)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${MPI_CXX_LIBRARIES} ${MPI_CXX_LINK_FLAGS})
else ()
set(onnxruntime_USE_NCCL OFF)
set(onnxruntime_USE_MPI OFF)
message( WARNING "MPI is not found. Please define onnxruntime_MPI_HOME to specify the path of MPI. Otherwise, NCCL will be disabled." )
if (onnxruntime_USE_MPI OR onnxruntime_USE_NCCL)
if (MPI_CXX_FOUND)
message( STATUS "MPI Version: ${MPI_CXX_VERSION}")
message( STATUS "MPI (include: ${MPI_CXX_INCLUDE_DIRS}, library: ${MPI_CXX_LIBRARIES})" )
mark_as_advanced(MPI_CXX_INCLUDE_DIRS MPI_CXX_LIBRARIES)
list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${MPI_CXX_LIBRARIES} ${MPI_CXX_LINK_FLAGS})
else ()
message(
FATAL_ERROR
"MPI is not found. Please define onnxruntime_MPI_HOME to specify the path of MPI. Otherwise, NCCL will be disabled."
)
endif()
endif()

# Find NCCL and MPI
if (onnxruntime_USE_NCCL AND MPI_CXX_FOUND)
if (onnxruntime_USE_NCCL)
if (onnxruntime_USE_CUDA)
set(NCCL_LIBNAME "nccl")
elseif (onnxruntime_USE_ROCM)
Expand Down Expand Up @@ -1436,13 +1439,15 @@ if (onnxruntime_ENABLE_TRAINING)
add_definitions(-DORT_USE_NCCL=1)
message( STATUS "NCCL is enabled in Linux GPU Build." )
else ()
set(onnxruntime_USE_NCCL OFF)
message( WARNING "NCCL is not found. Please use --nccl_home to specify the path of NCCL. Otherwise, NCCL is disabled." )
message(
FATAL_ERROR
"NCCL is not found. Please use --nccl_home to specify the path of NCCL. Otherwise, NCCL is disabled."
)
endif()
endif()
endif()

if (onnxruntime_USE_MPI AND MPI_CXX_FOUND)
if (onnxruntime_USE_MPI)
add_definitions(-DUSE_MPI=1)
endif()

Expand Down
6 changes: 4 additions & 2 deletions tools/ci_build/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,10 +192,12 @@ def convert_arg_line_to_args(self, arg_line):
parser.add_argument("--enable_training_apis", action="store_true", help="Enable ort training apis.")
parser.add_argument("--enable_training_ops", action="store_true", help="Enable training ops in inference graph.")

parser.add_argument("--disable_nccl", action="store_true", help="Disable Nccl.")
parser.add_argument("--disable_nccl", action="store_false", help="Disable NCCL, by default NCCL is disabled.")
parser.add_argument("--mpi_home", help="Path to MPI installation dir")
parser.add_argument("--nccl_home", help="Path to NCCL installation dir")
parser.add_argument("--use_mpi", nargs="?", default=True, const=True, type=_str_to_bool)
parser.add_argument(
"--use_mpi", nargs="?", default=False, const=True, type=_str_to_bool, help="Disabled by default."
)

# enable ONNX tests
parser.add_argument(
Expand Down