From 36890a0461cf8b171ad94a04e8537c217236c3f6 Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Fri, 4 Oct 2024 10:07:40 -0500 Subject: [PATCH 1/4] Use CallFnAfter instead of CallOnConditionKeep (#3841) * Use CallFnAfter instead of CallOnConditionKeep * fix periodic for callonconditionkeep --- tests/charm++/periodic/periodic.C | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/charm++/periodic/periodic.C b/tests/charm++/periodic/periodic.C index 8caa2b8f1a..4ead6c47fa 100644 --- a/tests/charm++/periodic/periodic.C +++ b/tests/charm++/periodic/periodic.C @@ -34,7 +34,11 @@ class main : public CBase_main { delete msg; mProxy = thisProxy; + #ifdef CALL_FN_AFTER startTime = CkWallTimer(); + #else + startTime = 0.0; + #endif gProxy = CProxy_testGroup::ckNew(COUNTER_MAX); CkPrintf("Testing Converse periodic callbacks on %d PEs for %d seconds\n", CkNumPes(), COUNTER_MAX); From 3305b9186c8bbbc5e8de89d0ac2345490c38a916 Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Fri, 4 Oct 2024 14:10:13 -0500 Subject: [PATCH 2/4] Ifdef to if typo fix (#3844) * Revert "Use CallFnAfter instead of CallOnConditionKeep (#3841)" This reverts commit 36890a0461cf8b171ad94a04e8537c217236c3f6. * ifdef to if --- tests/charm++/periodic/periodic.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/charm++/periodic/periodic.C b/tests/charm++/periodic/periodic.C index 4ead6c47fa..f0acf7c59a 100644 --- a/tests/charm++/periodic/periodic.C +++ b/tests/charm++/periodic/periodic.C @@ -34,7 +34,7 @@ class main : public CBase_main { delete msg; mProxy = thisProxy; - #ifdef CALL_FN_AFTER + #if CALL_FN_AFTER startTime = CkWallTimer(); #else startTime = 0.0; From d17744e46feadc22c8613c1e2d57374ca935209c Mon Sep 17 00:00:00 2001 From: Maya Taylor <70495835+mayantaylor@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:53:46 -0500 Subject: [PATCH 3/4] Fixing cmake charm++ and charm4py build with cuda (#3836) * fixing cmake cuda build: renaming cuda cmake option * linking cudart into charmlib.so --------- Co-authored-by: Ritvik Rao --- CMakeLists.txt | 13 ++++++++----- buildcmake | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b6f699574..50fcd28a0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -209,8 +209,7 @@ endif() option(BUILD_SHARED "Build Charm++ dynamic libraries" OFF) # Other options -option(CUDA "Build with CUDA support" OFF) - +option(BUILD_CUDA "Build with CUDA support" OFF) option(PXSHM "Build with PXSHM" OFF) # LRTS PMI options @@ -224,8 +223,6 @@ string(REPLACE ";" " " MY_EXTRA_OPTS "${MY_EXTRA_OPTS}") set(OPTS "${OPTS} ${MY_EXTRA_OPTS}") set(OPTSATBUILDTIME "${OPTSATBUILDTIME} ${MY_EXTRA_OPTS}") -# We need both BUILD_CUDA and CUDA -set(BUILD_CUDA ${CUDA}) # Also build shared Charm++ libraries in lib_so/ if(BUILD_SHARED) @@ -689,7 +686,8 @@ configure_file(src/scripts/conv-config.sh include/ COPYONLY) configure_file(src/arch/${VDIR}/conv-mach.sh include/ COPYONLY) set(CUDA_DIR "") -if(CUDA) +if(BUILD_CUDA) + file(GLOB_RECURSE hybridAPI-h-sources ${CMAKE_SOURCE_DIR}/src/arch/cuda/*.h) file(GLOB_RECURSE hybridAPI-cxx-sources ${CMAKE_SOURCE_DIR}/src/arch/cuda/*.cpp) foreach(file ${hybridAPI-h-sources}) @@ -995,6 +993,10 @@ if(${TARGET} STREQUAL "charm4py") target_link_libraries(charm ck converse memory-default threads-default ldb-rand "-Llib/ -standalone -whole-archive -c++stl -shared") endif() + if (${BUILD_CUDA}) + target_link_libraries(charm cudart cudahybridapi) + endif() + add_dependencies(charm hwloc) endif() @@ -1096,6 +1098,7 @@ foreach(l CUDA_DIR BUILD_CUDA CMK_AMPI_WITH_ROMIO CMK_MACOSX CMK_BUILD_PYTHON endforeach(l) # Add options +set(CUDA ${BUILD_CUDA}) # need CUDA to match conv-mach file name foreach(opt SMP OMP TCP PTHREADS SYNCFT PXSHM PERSISTENT OOC CUDA PAPI CXI) if(${opt}) string(TOLOWER ${opt} optl) diff --git a/buildcmake b/buildcmake index ab65c2407b..e7071b7fe4 100755 --- a/buildcmake +++ b/buildcmake @@ -647,7 +647,7 @@ CC=$opt_CC CXX=$opt_CXX FC=$opt_FC cmake "$my_srcdir" \ -DCCS="$opt_ccs" \ -DCHARMDEBUG="$opt_charmdebug" \ -DCONTROLPOINT="$opt_controlpoint" \ - -DCUDA="$opt_cuda" \ + -DBUILD_CUDA="$opt_cuda" \ -DDISABLE_TLS="$opt_disabletls" \ -DDRONE_MODE="$opt_drone_mode" \ -DENABLE_FORTRAN=$opt_enable_fortran \ From 55b1360f5ff023492027bb7d39f8a64701d6551c Mon Sep 17 00:00:00 2001 From: Thomas Quinn Date: Fri, 4 Oct 2024 13:53:22 -0700 Subject: [PATCH 4/4] Port verbs layer to verbs-linux-arm8 (#3837) * getInfiCmiChunkThread(): Check for empty PCQueue. Be more informative about ibv_reg_mr() failures. * Add verbs-linux-arm8 port. --------- Co-authored-by: Eric Bohm Co-authored-by: Ritvik Rao --- src/arch/verbs-linux-arm8/conv-mach-ibud.h | 23 +++++++ src/arch/verbs-linux-arm8/conv-mach-ibud.sh | 18 +++++ src/arch/verbs-linux-arm8/conv-mach-pxshm.h | 3 + src/arch/verbs-linux-arm8/conv-mach-pxshm.sh | 2 + src/arch/verbs-linux-arm8/conv-mach-smp.h | 20 ++++++ src/arch/verbs-linux-arm8/conv-mach-smp.sh | 3 + src/arch/verbs-linux-arm8/conv-mach.h | 72 ++++++++++++++++++++ src/arch/verbs-linux-arm8/conv-mach.sh | 24 +++++++ src/arch/verbs/machine-ibverbs.C | 17 ++++- 9 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 src/arch/verbs-linux-arm8/conv-mach-ibud.h create mode 100644 src/arch/verbs-linux-arm8/conv-mach-ibud.sh create mode 100644 src/arch/verbs-linux-arm8/conv-mach-pxshm.h create mode 100644 src/arch/verbs-linux-arm8/conv-mach-pxshm.sh create mode 100644 src/arch/verbs-linux-arm8/conv-mach-smp.h create mode 100644 src/arch/verbs-linux-arm8/conv-mach-smp.sh create mode 100644 src/arch/verbs-linux-arm8/conv-mach.h create mode 100644 src/arch/verbs-linux-arm8/conv-mach.sh diff --git a/src/arch/verbs-linux-arm8/conv-mach-ibud.h b/src/arch/verbs-linux-arm8/conv-mach-ibud.h new file mode 100644 index 0000000000..e9f7c9afb7 --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach-ibud.h @@ -0,0 +1,23 @@ +#undef CMK_USE_IBUD +#define CMK_USE_IBUD 1 + +#undef CMK_USE_IBVERBS +#define CMK_USE_IBVERBS 1 + +// FIXME: See if I need to include any of these flags +#undef CMK_NETPOLL +#define CMK_NETPOLL 1 + +#undef CMK_MALLOC_USE_GNU_MALLOC +#define CMK_MALLOC_USE_GNU_MALLOC 0 + +#undef CMK_MALLOC_USE_OS_BUILTIN +#define CMK_MALLOC_USE_OS_BUILTIN 1 + +#undef CMK_IMMEDIATE_MSG +#define CMK_IMMEDIATE_MSG 0 + +#undef CMK_DISABLE_SYNC +#define CMK_DISABLE_SYNC 1 + + diff --git a/src/arch/verbs-linux-arm8/conv-mach-ibud.sh b/src/arch/verbs-linux-arm8/conv-mach-ibud.sh new file mode 100644 index 0000000000..7aa875d72a --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach-ibud.sh @@ -0,0 +1,18 @@ +#default ibverbs path for openib +if test -z "$CMK_INCDIR" +then + #openib-1.1 + if test -f /opt/ofed/include/infiniband/verbs.h + then + CMK_INCDIR="-I/opt/ofed/include/" + CMK_LIBDIR="-L/opt/ofed/lib64" + fi + if test -f /usr/local/ofed/include/infiniband/verbs.h + then + CMK_INCDIR="-I/usr/local/ofed/include/" + CMK_LIBDIR="-L/usr/local/ofed/lib64" + fi +fi + +CMK_LIBS="$CMK_LIBS -libverbs" + diff --git a/src/arch/verbs-linux-arm8/conv-mach-pxshm.h b/src/arch/verbs-linux-arm8/conv-mach-pxshm.h new file mode 100644 index 0000000000..62c83b3e1c --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach-pxshm.h @@ -0,0 +1,3 @@ +#undef CMK_USE_PXSHM +#define CMK_USE_PXSHM 1 +#define PXSHM_LOCK 1 diff --git a/src/arch/verbs-linux-arm8/conv-mach-pxshm.sh b/src/arch/verbs-linux-arm8/conv-mach-pxshm.sh new file mode 100644 index 0000000000..876d88471c --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach-pxshm.sh @@ -0,0 +1,2 @@ +CMK_SYSLIBS="$CMK_SYSLIBS -lrt -lpthread" + diff --git a/src/arch/verbs-linux-arm8/conv-mach-smp.h b/src/arch/verbs-linux-arm8/conv-mach-smp.h new file mode 100644 index 0000000000..5e89297ba1 --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach-smp.h @@ -0,0 +1,20 @@ + +#define CMK_SMP 1 + + +#undef CMK_SHARED_VARS_UNAVAILABLE +#undef CMK_SHARED_VARS_POSIX_THREADS_SMP +#define CMK_SHARED_VARS_UNAVAILABLE 0 +#define CMK_SHARED_VARS_POSIX_THREADS_SMP 1 + +#undef CMK_MALLOC_USE_GNU_MALLOC +#undef CMK_MALLOC_USE_OS_BUILTIN +#define CMK_MALLOC_USE_GNU_MALLOC 0 +#define CMK_MALLOC_USE_OS_BUILTIN 1 + +/*#define CMK_MMAP_PROBE 1 */ + +/*#define CMK_PCQUEUE_LOCK 1 */ +/*#define CMK_USE_MFENCE 1 */ +/* Replaced by CMK_NOT_USE_TLS_THREAD as default */ +/*#define CMK_USE_TLS_THREAD 1*/ diff --git a/src/arch/verbs-linux-arm8/conv-mach-smp.sh b/src/arch/verbs-linux-arm8/conv-mach-smp.sh new file mode 100644 index 0000000000..ce625d65e9 --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach-smp.sh @@ -0,0 +1,3 @@ +CMK_DEFS="$CMK_DEFS -D_REENTRANT" +CMK_LIBS="-lpthread $CMK_LIBS " +CMK_SMP='1' diff --git a/src/arch/verbs-linux-arm8/conv-mach.h b/src/arch/verbs-linux-arm8/conv-mach.h new file mode 100644 index 0000000000..36024de3dd --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach.h @@ -0,0 +1,72 @@ + +#ifndef _CONV_MACH_H +#define _CONV_MACH_H + +#define CMK_USE_IBVERBS 1 +#define CMK_NETPOLL 1 + +#define CMK_ARM 1 + +#define CMK_ASYNC_NOT_NEEDED 0 +#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN 0 +#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP 0 +#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN 0 +#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN 1 + +#define CMK_DLL_CC "g++ -shared -O3 -o " + +#define CMK_GETPAGESIZE_AVAILABLE 1 + +#define CMK_MALLOC_USE_GNU_MALLOC 0 +#define CMK_MALLOC_USE_OS_BUILTIN 1 + +#define CMK_MEMORY_PAGESIZE 4096 +#define CMK_MEMORY_PROTECTABLE 0 + + +#define CMK_SSH_IS_A_COMMAND 1 +#define CMK_SSH_NOT_NEEDED 0 + +#define CMK_SHARED_VARS_UNAVAILABLE 1 + +#define CMK_THREADS_USE_CONTEXT 0 +#define CMK_THREADS_USE_FCONTEXT 1 +#define CMK_THREADS_USE_PTHREADS 0 +#define CMK_THREADS_ARE_WIN32_FIBERS 0 + +#define CMK_SIGNAL_NOT_NEEDED 0 +#define CMK_SIGNAL_USE_SIGACTION 0 +#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART 1 + +#define CMK_THREADS_REQUIRE_NO_CPV 0 +#define CMK_THREADS_COPY_STACK 0 + +#define CMK_TIMER_USE_RDTSC 0 +#define CMK_TIMER_USE_GETRUSAGE 1 +#define CMK_TIMER_USE_SPECIAL 0 +#define CMK_TIMER_USE_TIMES 0 + + +#define CMK_64BIT 1 + +#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT 0 +#define CMK_WHEN_PROCESSOR_IDLE_USLEEP 1 + + +#define CMK_DEBUG_MODE 0 +#define CMK_WEB_MODE 1 + +#define CMK_LBDB_ON 1 + +/* +#undef CMK_IMMEDIATE_MSG +#define CMK_IMMEDIATE_MSG 0 +*/ + +#undef CMK_DISABLE_SYNC +#define CMK_DISABLE_SYNC 1 + +#undef CMK_IBVERBS_FAST_START +#define CMK_IBVERBS_FAST_START 0 + +#endif diff --git a/src/arch/verbs-linux-arm8/conv-mach.sh b/src/arch/verbs-linux-arm8/conv-mach.sh new file mode 100644 index 0000000000..c058cdc5ab --- /dev/null +++ b/src/arch/verbs-linux-arm8/conv-mach.sh @@ -0,0 +1,24 @@ +. $CHARMINC/cc-gcc.sh + +#CMK_DEFS="$CMK_DEFS -DHAVE_USR_INCLUDE_MALLOC_H=1 " +CMK_XIOPTS='' +CMK_LIBS="$CMK_LIBS -libverbs" +CMK_CXX_OPTIMIZE='-O3' + +CMK_QT='generic64-light' + +#default ibverbs path for openib +if test -z "$CMK_INCDIR" +then + #openib-1.1 + if test -f /opt/ofed/include/infiniband/verbs.h + then + CMK_INCDIR='-I/opt/ofed/include/' + CMK_LIBDIR='-L/opt/ofed/lib64' + fi + if test -f /usr/local/ofed/include/infiniband/verbs.h + then + CMK_INCDIR='-I/usr/local/ofed/include/' + CMK_LIBDIR='-L/usr/local/ofed/lib64' + fi +fi diff --git a/src/arch/verbs/machine-ibverbs.C b/src/arch/verbs/machine-ibverbs.C index 76c6dad78d..835ee47f73 100644 --- a/src/arch/verbs/machine-ibverbs.C +++ b/src/arch/verbs/machine-ibverbs.C @@ -2540,7 +2540,9 @@ static inline void *getInfiCmiChunkThread(int dataSize){ if(!PCQueueEmpty(queuePool[CmiMyRank()][i])){ for(j = 0; j < PCQueueLength(queuePool[CmiMyRank()][i]); j++){ pointer = (void *)PCQueuePop(queuePool[CmiMyRank()][i]); - infi_CmiFreeDirect(pointer); + // Check because queue might actually be empty + if(pointer != NULL) + infi_CmiFreeDirect(pointer); } } } @@ -2570,8 +2572,12 @@ static inline void *getInfiCmiChunkThread(int dataSize){ hdr = (CmiChunkHeader *)res; key = ibv_reg_mr(context->pd,res,(allocSize+sizeof(CmiChunkHeader))*count,IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - if(key == NULL) + if(key == NULL) { + CmiError("Failed to allocate %d messages in pool %d\n", count, poolIdx); + CmiError("Consider increasing IBVBlockAllocRatio (currently %d) or IBVBlockThreshold (currently %d)\n", + blockAllocRatio, blockThreshold); CmiAbort("ibv_reg_mr failed to pin memory\n"); + } #if CMK_IBVERBS_STATS numCurReg++; numReg++; @@ -2672,7 +2678,12 @@ static inline void *getInfiCmiChunk(int dataSize){ hdr = (CmiChunkHeader *)res; key = ibv_reg_mr(context->pd,res,(allocSize+sizeof(CmiChunkHeader))*count,IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - CmiAssert(key != NULL); + if(key == NULL) { + CmiError("Failed to allocate %d messages in pool %d\n", count, poolIdx); + CmiError("Consider increasing IBVBlockAllocRatio (currently %d) or IBVBlockThreshold (currently %d)\n", + blockAllocRatio, blockThreshold); + CmiAbort("ibv_reg_mr failed to pin memory\n"); + } #if CMK_IBVERBS_STATS numCurReg++; numReg++;