Skip to content

Commit

Permalink
Support for shared libraries in GPU execution (python launch support) (…
Browse files Browse the repository at this point in the history 
…BlueBrain/CoreNeuron#795)

* coreneuron and mechanism library can be built as shared and it
  enables launching coreneuron on GPU via python
* update MOD2C and NMODL fixes to handle GLOBAL variables
      See BlueBrain/mod2c/pull/78
      See BlueBrain/nmodl/pull/904
* removed acc/openmp global annotations for celsius, pi and secondorder
  and they don't need to be copied on GPU
* Pass Memb_list* as an argument for all common prototypes in order
   to support global variables via argument
* free ml->instance if not empty
* add link to libscopmath in neuron as well
* nrn_ghk is now declared inline.
* homegrown present table to avoid dynamic loading + acc_deviceptr limitations
* use -gpu=nordc and make #pragma acc routine seq functions inline
* drop -lscopmath as its folded in elsewhere
* random123 header reorganisation
* try and cleanup CLI11 handling.
* try and consolidate build logic
* some CORENEURON_ -> CORENRN_ for consistency.
* export OpenACC flags to NEURON separately as well as part
     of the whole ... -lcoreneuron ... link line.
* libcoreneuron.so -> libcorenrnmech.so, try and fix static builds
* do not enable OpenMP in shared/OpenACC builds.
* add rpaths inside nrnivmodl-core.
* accept a private destructor function pointer from generated mechanisms
* drop ${TEST_EXEC_PREFIX} that was causing simple tests to be executed on many ranks.
* CORENEURON_GPU_DEBUG: add environment variable that enables cnrn_target_* debug messages.

fixes BlueBrain/CoreNeuron#141

Co-authored-by: Olli Lupton <[email protected]>

CoreNEURON Repo SHA: BlueBrain/CoreNeuron@12272f8
  • Loading branch information
@pramodk
pramodk authored Aug 28, 2022
1 parent eb88b59 commit 1120f53
Show file tree
Hide file tree
Showing 56 changed files with 1,133 additions and 744 deletions.
40 changes: 20 additions & 20 deletions bin/nrnivmodl_core_makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@ MOD_OBJS_DIR = $(OUTPUT_DIR)/corenrn/build

# Linked libraries gathered by CMake
LDFLAGS = $(LINKFLAGS) @CORENRN_COMMON_LDFLAGS@
CORENRNLIB_FLAGS = -L$(CORENRN_LIB_DIR) -lcoreneuron
CORENRNLIB_FLAGS += $(if @reportinglib_LIB_DIR@, -W$(subst ;, -W,l,-rpath,@reportinglib_LIB_DIR@),)
CORENRNLIB_FLAGS += $(if @sonatareport_LIB_DIR@, -W$(subst ;, -W,l,-rpath,@sonatareport_LIB_DIR@),)
CORENRNLIB_FLAGS += $(if @caliper_LIB_DIR@, -W$(subst ;, -W,l,-rpath,@caliper_LIB_DIR@),)
CORENRNLIB_FLAGS += $(if @caliper_LIB_DIR@,-L@caliper_LIB_DIR@,)

# Includes paths gathered by CMake
# coreneuron/utils/randoms goes first because it needs to override the NEURON
Expand Down Expand Up @@ -82,7 +77,7 @@ ifeq ($(wildcard $(CORENRN_PERLEXE)),)
endif

CXXFLAGS = @CORENRN_CXX_FLAGS@
CXX_COMPILE_CMD = $(CXX) $(CXXFLAGS) @CMAKE_CXX_COMPILE_OPTIONS_PIC@ @CORENRN_COMMON_COMPILE_DEFS@ $(INCLUDES)
CXX_COMPILE_CMD = $(CXX) $(CXXFLAGS) @CMAKE_CXX_COMPILE_OPTIONS_PIC@ $(INCLUDES)
CXX_LINK_EXE_CMD = $(CXX) $(CXXFLAGS) @CMAKE_EXE_LINKER_FLAGS@
CXX_SHARED_LIB_CMD = $(CXX) $(CXXFLAGS) @CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS@ @CMAKE_SHARED_LIBRARY_CXX_FLAGS@ @CMAKE_SHARED_LINKER_FLAGS@

Expand Down Expand Up @@ -207,34 +202,39 @@ endif


# main target to build binary
$(SPECIAL_EXE): coremech_lib_target
$(SPECIAL_EXE): $(corenrnmech_lib_target)
@printf " => $(C_GREEN)Binary$(C_RESET) creating $(SPECIAL_EXE)\n"
$(CXX_LINK_EXE_CMD) -o $(SPECIAL_EXE) $(CORENRN_SHARE_CORENRN_DIR)/coreneuron.cpp \
-I$(CORENRN_INC_DIR) $(INCFLAGS) \
-L$(OUTPUT_DIR) -l$(COREMECH_LIB_NAME) $(CORENRNLIB_FLAGS) $(LDFLAGS) \
-L$(OUTPUT_DIR) -l$(COREMECH_LIB_NAME) $(LDFLAGS) \
-L$(CORENRN_LIB_DIR) \
-Wl,-rpath,'$(LIB_RPATH)' -Wl,-rpath,$(CORENRN_LIB_DIR) -Wl,-rpath,'$(INSTALL_LIB_RPATH)'

coremech_lib_target: $(corenrnmech_lib_target)
rm -rf $(OUTPUT_DIR)/.libs/lib$(COREMECH_LIB_NAME)$(LIB_SUFFIX); \
mkdir -p $(OUTPUT_DIR)/.libs; \
ln -s ../lib$(COREMECH_LIB_NAME)$(LIB_SUFFIX) $(OUTPUT_DIR)/.libs/lib$(COREMECH_LIB_NAME)$(LIB_SUFFIX)

$(ENGINEMECH_OBJ): $(CORENRN_SHARE_CORENRN_DIR)/enginemech.cpp | $(MOD_OBJS_DIR)
$(CXX_COMPILE_CMD) -c -DADDITIONAL_MECHS $(CORENRN_SHARE_CORENRN_DIR)/enginemech.cpp -o $(ENGINEMECH_OBJ)

# build shared library of mechanisms
coremech_lib_shared: $(ALL_OBJS) $(ENGINEMECH_OBJ) build_always
# extract the object files from libcoreneuron-core.a
mkdir -p $(MOD_OBJS_DIR)/libcoreneuron-core
rm -f $(MOD_OBJS_DIR)/libcoreneuron-core/*.o
# --output is only supported by modern versions of ar
(cd $(MOD_OBJS_DIR)/libcoreneuron-core && ar x $(CORENRN_LIB_DIR)/libcoreneuron-core.a)
$(CXX_SHARED_LIB_CMD) $(ENGINEMECH_OBJ) -o ${COREMECH_LIB_PATH} $(ALL_OBJS) \
-I$(CORENRN_INC_DIR) $(INCFLAGS) \
$(LDFLAGS) $(CORENRN_LIB_DIR)/libscopmath.a \
${SONAME_OPTION} $(CORENRNLIB_FLAGS) -Wl,-rpath,$(CORENRN_LIB_DIR);
@CORENEURON_LINKER_START_GROUP@ \
$(MOD_OBJS_DIR)/libcoreneuron-core/*.o @CORENEURON_LINKER_END_GROUP@ \
$(LDFLAGS) ${SONAME_OPTION} \
-Wl,-rpath,$(CORENRN_LIB_DIR) -L$(CORENRN_LIB_DIR)
# cleanup
rm $(MOD_OBJS_DIR)/libcoreneuron-core/*.o

# build static library of mechanisms
coremech_lib_static: $(ALL_OBJS) $(ENGINEMECH_OBJ) build_always
mkdir -p $(MOD_OBJS_DIR)/scopmath; \
cd $(MOD_OBJS_DIR)/scopmath && ar -x $(CORENRN_LIB_DIR)/libscopmath.a && cd -;\
rm -f ${COREMECH_LIB_PATH}; \
ar cq ${COREMECH_LIB_PATH} $(ENGINEMECH_OBJ) $(ALL_OBJS) $(MOD_OBJS_DIR)/scopmath/*.o;
# make a libcorenrnmech.a by copying libcoreneuron-core.a and then appending
# the newly compiled objects
cp $(CORENRN_LIB_DIR)/libcoreneuron-core.a ${COREMECH_LIB_PATH}
ar r ${COREMECH_LIB_PATH} $(ENGINEMECH_OBJ) $(ALL_OBJS)

# compile cpp files to .o
$(MOD_OBJS_DIR)/%.o: $(MOD_TO_CPP_DIR)/%.cpp | $(MOD_OBJS_DIR)
Expand Down Expand Up @@ -273,7 +273,7 @@ $(MOD_OBJS_DIR):
mkdir -p $(MOD_OBJS_DIR)

# install binary and libraries
install: $(SPECIAL_EXE) coremech_lib_target
install: $(SPECIAL_EXE)
install -d $(DESTDIR)/bin $(DESTDIR)/lib
install ${COREMECH_LIB_PATH} $(DESTDIR)/lib
install $(SPECIAL_EXE) $(DESTDIR)/bin
Expand Down
100 changes: 36 additions & 64 deletions cmake/coreneuron/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,6 @@ set(CORENRN_ACCELERATOR_OFFLOAD "Disabled")
if(CORENRN_ENABLE_GPU)
# Older CMake versions than 3.15 have not been tested for GPU/CUDA/OpenACC support after
# https://github.com/BlueBrain/CoreNeuron/pull/609.
# https://cmake.org/cmake/help/latest/release/3.14.html#properties suggests there would be
# problems because of expressions like set_target_properties(lfp_test_bin PROPERTIES
# CUDA_RESOLVE_DEVICE_SYMBOLS OFF)

# Fail hard and early if we don't have the PGI/NVHPC compiler.
if(NOT CORENRN_HAVE_NVHPC_COMPILER)
Expand Down Expand Up @@ -263,17 +260,21 @@ find_package(Perl REQUIRED)
# Common build options
# =============================================================================
# build mod files for coreneuron
add_definitions(-DCORENEURON_BUILD)

list(APPEND CORENRN_COMPILE_DEFS CORENEURON_BUILD)
set(CMAKE_REQUIRED_QUIET TRUE)
check_include_files(malloc.h have_malloc_h)
if(have_malloc_h)
add_definitions("-DHAVE_MALLOC_H")
list(APPEND CORENRN_COMPILE_DEFS HAVE_MALLOC_H)
endif()

# =============================================================================
# Build option specific compiler flags
# =============================================================================
if(CORENRN_ENABLE_NMODL)
# We use Eigen for "small" matrices with thread-level parallelism handled at a higher level; tell
# Eigen not to try to multithread internally
list(APPEND CORENRN_COMPILE_DEFS EIGEN_DONT_PARALLELIZE)
endif()
if(CORENRN_HAVE_NVHPC_COMPILER)
# PGI with llvm code generation doesn't have necessary assembly intrinsic headers
list(APPEND CORENRN_COMPILE_DEFS EIGEN_DONT_VECTORIZE=1)
Expand All @@ -292,14 +293,6 @@ if(CORENRN_HAVE_NVHPC_COMPILER)
endif()
endif()

# ~~~
# OpenACC needs to build static library in order to have global/routines working.
# See https://www.pgroup.com/userforum/viewtopic.php?t=5350
# ~~~
if(CORENRN_ENABLE_GPU)
set(CORENRN_ENABLE_SHARED OFF)
endif()

if(CORENRN_ENABLE_SHARED)
set(COMPILE_LIBRARY_TYPE "SHARED")
else()
Expand All @@ -313,14 +306,14 @@ endif()

if(CORENRN_ENABLE_MPI)
find_package(MPI REQUIRED)
add_definitions("-DNRNMPI=1")
list(APPEND CORENRN_COMPILE_DEFS NRNMPI=1)
# avoid linking to C++ bindings
add_definitions("-DMPI_NO_CPPBIND=1")
add_definitions("-DOMPI_SKIP_MPICXX=1")
add_definitions("-DMPICH_SKIP_MPICXX=1")
list(APPEND CORENRN_COMPILE_DEFS MPI_NO_CPPBIND=1)
list(APPEND CORENRN_COMPILE_DEFS OMPI_SKIP_MPICXX=1)
list(APPEND CORENRN_COMPILE_DEFS MPICH_SKIP_MPICXX=1)
else()
add_definitions("-DNRNMPI=0")
add_definitions("-DNRN_MULTISEND=0")
list(APPEND CORENRN_COMPILE_DEFS NRNMPI=0)
list(APPEND CORENRN_COMPILE_DEFS NRN_MULTISEND=0)
endif()

if(CORENRN_ENABLE_OPENMP)
Expand All @@ -331,23 +324,23 @@ if(CORENRN_ENABLE_OPENMP)
endif()
endif()

add_definitions("-DLAYOUT=0")
list(APPEND CORENRN_COMPILE_DEFS LAYOUT=0)

if(NOT CORENRN_ENABLE_HOC_EXP)
add_definitions("-DDISABLE_HOC_EXP")
list(APPEND CORENRN_COMPILE_DEFS DISABLE_HOC_EXP)
endif()

# splay tree required for net_move
if(CORENRN_ENABLE_SPLAYTREE_QUEUING)
add_definitions("-DENABLE_SPLAYTREE_QUEUING")
list(APPEND CORENRN_COMPILE_DEFS ENABLE_SPLAYTREE_QUEUING)
endif()

if(NOT CORENRN_ENABLE_NET_RECEIVE_BUFFER)
add_definitions("-DNET_RECEIVE_BUFFERING=0")
list(APPEND CORENRN_COMPILE_DEFS NET_RECEIVE_BUFFERING=0)
endif()

if(NOT CORENRN_ENABLE_TIMEOUT)
add_definitions("-DDISABLE_TIMEOUT")
list(APPEND CORENRN_COMPILE_DEFS DISABLE_TIMEOUT)
endif()

if(CORENRN_ENABLE_REPORTING)
Expand All @@ -356,15 +349,15 @@ if(CORENRN_ENABLE_REPORTING)
find_program(H5DUMP_EXECUTABLE h5dump)

if(reportinglib_FOUND)
add_definitions("-DENABLE_BIN_REPORTS")
list(APPEND CORENRN_COMPILE_DEFS ENABLE_BIN_REPORTS)
set(ENABLE_BIN_REPORTS_TESTS ON)
else()
set(reportinglib_INCLUDE_DIR "")
set(reportinglib_LIBRARY "")
endif()
if(sonata_FOUND)
if(TARGET sonata::sonata_report)
add_definitions("-DENABLE_SONATA_REPORTS")
list(APPEND CORENRN_COMPILE_DEFS ENABLE_SONATA_REPORTS)
set(ENABLE_SONATA_REPORTS_TESTS ON)
else()
message(SEND_ERROR "SONATA library was found but without reporting support")
Expand All @@ -384,6 +377,7 @@ if(CORENRN_ENABLE_LEGACY_UNITS)
else()
set(CORENRN_USE_LEGACY_UNITS 0)
endif()
list(APPEND CORENRN_COMPILE_DEFS CORENEURON_USE_LEGACY_UNITS=${CORENRN_USE_LEGACY_UNITS})
# Propagate Legacy Units flag to backends.
set(MOD2C_ENABLE_LEGACY_UNITS
${CORENRN_ENABLE_LEGACY_UNITS}
Expand All @@ -396,7 +390,7 @@ if(CORENRN_ENABLE_MPI_DYNAMIC)
if(NOT CORENRN_ENABLE_MPI)
message(FATAL_ERROR "Cannot enable dynamic mpi without mpi")
endif()
add_compile_definitions(CORENRN_ENABLE_MPI_DYNAMIC)
list(APPEND CORENRN_COMPILE_DEFS CORENEURON_ENABLE_MPI_DYNAMIC)
endif()

if(CORENRN_ENABLE_PRCELLSTATE)
Expand All @@ -405,7 +399,7 @@ else()
set(CORENRN_NRN_PRCELLSTATE 0)
endif()
if(MINGW)
add_definitions("-DMINGW")
list(APPEND CORENRN_COMPILE_DEFS MINGW)
endif()

# =============================================================================
Expand Down Expand Up @@ -448,22 +442,20 @@ endif()
# =============================================================================
if(CORENRN_ENABLE_CALIPER_PROFILING)
find_package(caliper REQUIRED)
include_directories(${caliper_INCLUDE_DIR})
add_definitions("-DCORENEURON_CALIPER")
set(CALIPER_LIB "caliper")
set_property(GLOBAL APPEND_STRING PROPERTY CORENEURON_LIB_LINK_FLAGS
" -L${caliper_LIB_DIR} -l${CALIPER_LIB}")
list(APPEND CORENRN_COMPILE_DEFS CORENEURON_CALIPER)
set(CORENRN_CALIPER_LIB caliper)
endif()

if(CORENRN_ENABLE_LIKWID_PROFILING)
find_package(likwid REQUIRED)
list(APPEND CORENRN_COMPILE_DEFS LIKWID_PERFMON)
# TODO: avoid this part, probably by using some likwid CMake target
include_directories(${likwid_INCLUDE_DIRS})
add_definitions("-DLIKWID_PERFMON")
endif()

# enable debugging code with extra logs to stdout
if(CORENRN_ENABLE_DEBUG_CODE)
add_definitions(-DCORENRN_DEBUG -DCHKPNTDEBUG -DCORENRN_DEBUG_QUEUE -DINTERLEAVE_DEBUG)
list(APPEND CORENRN_COMPILE_DEFS CORENRN_DEBUG CHKPNTDEBUG CORENRN_DEBUG_QUEUE INTERLEAVE_DEBUG)
endif()

# =============================================================================
Expand All @@ -473,38 +465,18 @@ endif()
# compiler will be invoked with these flags, so we have to use flags that are as generic as
# possible.
if(NOT DEFINED NRN_WHEEL_BUILD OR NOT NRN_WHEEL_BUILD)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${IGNORE_UNKNOWN_PRAGMA_FLAGS}")
list(APPEND CORENRN_EXTRA_CXX_FLAGS "${IGNORE_UNKNOWN_PRAGMA_FLAGS}")
endif()

# =============================================================================
# Add main directories
# =============================================================================
# Add the main source directory
add_subdirectory(coreneuron)

if(CORENRN_ENABLE_GPU)
get_target_property(CORENRN_LINK_LIBRARIES coreneuron INTERFACE_LINK_LIBRARIES)
if(CORENRN_LINK_LIBRARIES)
foreach(LIB ${CORENRN_LINK_LIBRARIES})
get_filename_component(dir_path ${LIB} DIRECTORY)
if(TARGET ${LIB})
# See, for example, caliper where the coreneuron target depends on the caliper target (so we
# get LIB=caliper in this loop), but -l and -L are already added manually here:
# https://github.com/BlueBrain/CoreNeuron/blob/856cea4aa647c8f2b0d5bda6d0fc32144c5942e3/CMakeLists.txt#L411-L412
message(
NOTICE
"Ignoring dependency '${LIB}' of 'coreneuron' and assuming relevant flags have already been added to CORENEURON_LIB_LINK_FLAGS."
)
elseif(NOT dir_path)
# In case LIB is not a target but is just the name of a library, e.g. "dl"
set_property(GLOBAL APPEND_STRING PROPERTY CORENEURON_LIB_LINK_FLAGS " -l${LIB}")
else()
set_property(GLOBAL APPEND_STRING PROPERTY CORENEURON_LIB_LINK_FLAGS " ${LIB}")
endif()
endforeach()
endif()
endif()

# Extract the various compiler option strings to use inside nrnivmodl-core. Sets the global property
# CORENRN_LIB_LINK_FLAGS, which contains the arguments that must be added to the link line for
# `special` to link against `libcorenrnmech.{a,so}`
include(MakefileBuildOptions)

# Generate the nrnivmodl-core script and makefile using the options from MakefileBuildOptions
add_subdirectory(extra)

if(CORENRN_ENABLE_UNIT_TESTS)
Expand All @@ -514,7 +486,7 @@ endif()
# =============================================================================
# Install cmake modules
# =============================================================================
get_property(CORENEURON_LIB_LINK_FLAGS GLOBAL PROPERTY CORENEURON_LIB_LINK_FLAGS)
get_property(CORENRN_NEURON_LINK_FLAGS GLOBAL PROPERTY CORENRN_NEURON_LINK_FLAGS)
configure_file(CMake/coreneuron-config.cmake.in CMake/coreneuron-config.cmake @ONLY)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/CMake/coreneuron-config.cmake" DESTINATION share/cmake)
install(EXPORT coreneuron DESTINATION share/cmake)
Expand Down
Loading

0 comments on commit 1120f53

Please sign in to comment.