From 802409338b6cf634ffcbbf77b1ddba9b6669a856 Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Tue, 5 Jan 2021 00:53:46 +0200 Subject: [PATCH 1/2] CUDA/TEST: Check if libcudart_static is present for memory hook test --- config/m4/cuda.m4 | 24 +++++++++++++++++++----- configure.ac | 1 + contrib/test_jenkins.sh | 30 +++++++++++++++++++----------- test/apps/Makefile.am | 8 +++++--- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/config/m4/cuda.m4 b/config/m4/cuda.m4 index 1862eb6148c..1e9a8ea31e2 100644 --- a/config/m4/cuda.m4 +++ b/config/m4/cuda.m4 @@ -12,7 +12,10 @@ AS_IF([test "x$cuda_checked" != "xyes"], [], [with_cuda=guess]) AS_IF([test "x$with_cuda" = "xno"], - [cuda_happy=no], + [ + cuda_happy=no + have_cuda_static=no + ], [ save_CPPFLAGS="$CPPFLAGS" save_LDFLAGS="$LDFLAGS" @@ -40,11 +43,21 @@ AS_IF([test "x$cuda_checked" != "xyes"], # Check cuda libraries AS_IF([test "x$cuda_happy" = "xyes"], - [AC_CHECK_LIB([cuda], [cuDeviceGetUuid], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcuda"], [cuda_happy="no"])]) + [AC_CHECK_LIB([cuda], [cuDeviceGetUuid], + [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcuda"], [cuda_happy="no"])]) + AS_IF([test "x$cuda_happy" = "xyes"], + [AC_CHECK_LIB([cudart], [cudaGetDeviceCount], + [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcudart"], [cuda_happy="no"])]) + + LDFLAGS="$save_LDFLAGS" + + # Check for cuda static library + have_cuda_static="no" AS_IF([test "x$cuda_happy" = "xyes"], - [AC_CHECK_LIB([cudart], [cudaGetDeviceCount], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcudart"], [cuda_happy="no"])]) + [AC_CHECK_LIB([cudart_static], [cudaGetDeviceCount], + [AC_SUBST([CUDA_STATIC_LIB], [-lcudart_static]) + have_cuda_static="yes"], + [], [-ldl -lrt -lpthread])]) CPPFLAGS="$save_CPPFLAGS" LDFLAGS="$save_LDFLAGS" @@ -61,6 +74,7 @@ AS_IF([test "x$cuda_checked" != "xyes"], cuda_checked=yes AM_CONDITIONAL([HAVE_CUDA], [test "x$cuda_happy" != xno]) + AM_CONDITIONAL([HAVE_CUDA_STATIC], [test "X$have_cuda_static" = "Xyes"]) ]) # "x$cuda_checked" != "xyes" diff --git a/configure.ac b/configure.ac index b6f1db0cb4d..426916bad72 100644 --- a/configure.ac +++ b/configure.ac @@ -185,6 +185,7 @@ AS_IF([test "x$with_docs_only" = xyes], AM_CONDITIONAL([HAVE_TL_CM], [false]) AM_CONDITIONAL([HAVE_CRAY_UGNI], [false]) AM_CONDITIONAL([HAVE_CUDA], [false]) + AM_CONDITIONAL([HAVE_CUDA_STATIC], [false]) AM_CONDITIONAL([HAVE_GDR_COPY], [false]) AM_CONDITIONAL([HAVE_ROCM], [false]) AM_CONDITIONAL([HAVE_HIP], [false]) diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh index 90af015b124..39c1a8d5898 100755 --- a/contrib/test_jenkins.sh +++ b/contrib/test_jenkins.sh @@ -1365,28 +1365,36 @@ test_malloc_hook() { if [ "X$have_cuda" == "Xyes" ] then + cuda_dynamic_exe=./test/apps/test_cuda_hook_dynamic + cuda_static_exe=./test/apps/test_cuda_hook_static + for mode in reloc bistro do export UCX_MEM_CUDA_HOOK_MODE=${mode} # Run cuda memory hooks with dynamic link - ./test/apps/test_cuda_hook_dynamic + ${cuda_dynamic_exe} - # Run cuda memory hooks with static link - ./test/apps/test_cuda_hook_static && status="pass" || status="fail" - [ ${mode} == "bistro" ] && exp_status="pass" || exp_status="fail" - if [ ${status} == ${exp_status} ] + # Run cuda memory hooks with static link, if exists. If the static + # library 'libcudart_static.a' is not present, static test will not + # be built. + if [ -x ${cuda_static_exe} ] then - echo "Static link with cuda ${status}, as expected" - else - echo "Static link with cuda is expected to ${exp_status}, actual: ${status}" - exit 1 + ${cuda_static_exe} && status="pass" || status="fail" + [ ${mode} == "bistro" ] && exp_status="pass" || exp_status="fail" + if [ ${status} == ${exp_status} ] + then + echo "Static link with cuda ${status}, as expected" + else + echo "Static link with cuda is expected to ${exp_status}, actual: ${status}" + exit 1 + fi fi # Test that driver API hooks work in both reloc and bistro modes, # since we call them directly from the test - ./test/apps/test_cuda_hook_dynamic -d - ./test/apps/test_cuda_hook_static -d + ${cuda_dynamic_exe} -d + [ -x ${cuda_static_exe} ] && ${cuda_static_exe} -d unset UCX_MEM_CUDA_HOOK_MODE done diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am index 2ada50f7bef..0a413f7af63 100644 --- a/test/apps/Makefile.am +++ b/test/apps/Makefile.am @@ -55,8 +55,7 @@ test_init_mt_CFLAGS = $(BASE_CFLAGS) $(OPENMP_CFLAGS) test_init_mt_LDADD = $(top_builddir)/src/ucp/libucp.la if HAVE_CUDA -noinst_PROGRAMS += test_cuda_hook_dynamic \ - test_cuda_hook_static +noinst_PROGRAMS += test_cuda_hook_dynamic test_cuda_hook_dynamic_SOURCES = test_cuda_hook.c test_cuda_hook_dynamic_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) @@ -65,15 +64,18 @@ test_cuda_hook_dynamic_LDFLAGS = $(CUDA_LDFLAGS) test_cuda_hook_dynamic_LDADD = $(top_builddir)/src/ucp/libucp.la \ $(top_builddir)/src/ucm/libucm.la +if HAVE_CUDA_STATIC noinst_PROGRAMS += test_cuda_hook_static test_cuda_hook_static_SOURCES = test_cuda_hook.c test_cuda_hook_static_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) test_cuda_hook_static_CFLAGS = $(BASE_CFLAGS) -test_cuda_hook_static_LDFLAGS = -lcudart_static -lcuda -lrt -ldl -lpthread +test_cuda_hook_static_LDFLAGS = $(CUDA_STATIC_LIB) -lcuda -lrt -ldl -lpthread test_cuda_hook_static_LDADD = $(top_builddir)/src/ucp/libucp.la \ $(top_builddir)/src/ucm/libucm.la endif +endif + if HAVE_TCMALLOC noinst_PROGRAMS += test_tcmalloc test_tcmalloc_SOURCES = test_tcmalloc.c From c01b78f2f9296d457451bd65c5c2533f9bd067ce Mon Sep 17 00:00:00 2001 From: Yossi Itigin Date: Wed, 6 Jan 2021 02:35:44 +0200 Subject: [PATCH 2/2] BUILD/CUDA: Separate CUDA_LIBS from CUDA_LDFLAGS Allows passing Cuda -L flag to static build example, without passing -lcudart. --- config/m4/cuda.m4 | 12 +++++++++--- examples/Makefile.am | 14 +++++++++----- src/tools/perf/cuda/Makefile.am | 1 + src/ucm/cuda/Makefile.am | 2 +- src/uct/cuda/Makefile.am | 5 +++-- src/uct/cuda/gdr_copy/Makefile.am | 6 ++++-- test/apps/Makefile.am | 8 +++++--- test/gtest/Makefile.am | 6 ++++-- 8 files changed, 36 insertions(+), 18 deletions(-) diff --git a/config/m4/cuda.m4 b/config/m4/cuda.m4 index 1e9a8ea31e2..74bf0e56435 100644 --- a/config/m4/cuda.m4 +++ b/config/m4/cuda.m4 @@ -19,9 +19,12 @@ AS_IF([test "x$cuda_checked" != "xyes"], [ save_CPPFLAGS="$CPPFLAGS" save_LDFLAGS="$LDFLAGS" + save_LIBS="$LIBS" CUDA_CPPFLAGS="" CUDA_LDFLAGS="" + CUDA_LIBS="" + CUDA_STATIC_LIBS="" AS_IF([test ! -z "$with_cuda" -a "x$with_cuda" != "xyes" -a "x$with_cuda" != "xguess"], [ucx_check_cuda_dir="$with_cuda" @@ -44,10 +47,10 @@ AS_IF([test "x$cuda_checked" != "xyes"], # Check cuda libraries AS_IF([test "x$cuda_happy" = "xyes"], [AC_CHECK_LIB([cuda], [cuDeviceGetUuid], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcuda"], [cuda_happy="no"])]) + [CUDA_LIBS="$CUDA_LIBS -lcuda"], [cuda_happy="no"])]) AS_IF([test "x$cuda_happy" = "xyes"], [AC_CHECK_LIB([cudart], [cudaGetDeviceCount], - [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcudart"], [cuda_happy="no"])]) + [CUDA_LIBS="$CUDA_LIBS -lcudart"], [cuda_happy="no"])]) LDFLAGS="$save_LDFLAGS" @@ -55,16 +58,19 @@ AS_IF([test "x$cuda_checked" != "xyes"], have_cuda_static="no" AS_IF([test "x$cuda_happy" = "xyes"], [AC_CHECK_LIB([cudart_static], [cudaGetDeviceCount], - [AC_SUBST([CUDA_STATIC_LIB], [-lcudart_static]) + [CUDA_STATIC_LIBS="$CUDA_STATIC_LIBS -lcudart_static" have_cuda_static="yes"], [], [-ldl -lrt -lpthread])]) CPPFLAGS="$save_CPPFLAGS" LDFLAGS="$save_LDFLAGS" + LIBS="$save_LIBS" AS_IF([test "x$cuda_happy" = "xyes"], [AC_SUBST([CUDA_CPPFLAGS], ["$CUDA_CPPFLAGS"]) AC_SUBST([CUDA_LDFLAGS], ["$CUDA_LDFLAGS"]) + AC_SUBST([CUDA_LIBS], ["$CUDA_LIBS"]) + AC_SUBST([CUDA_STATIC_LIBS], ["$CUDA_STATIC_LIBS"]) AC_DEFINE([HAVE_CUDA], 1, [Enable CUDA support])], [AS_IF([test "x$with_cuda" != "xguess"], [AC_MSG_ERROR([CUDA support is requested but cuda packages cannot be found])], diff --git a/examples/Makefile.am b/examples/Makefile.am index 05cde2765dd..318142f331c 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -13,18 +13,20 @@ dist_examples_DATA = \ ucp_client_server.c if HAVE_CUDA -EXAMPLE_CUDA_LDFLAGS = $(CUDA_LDFLAGS) +EXAMPLE_CUDA_LD_FLAGS = $(CUDA_LDFLAGS) +EXAMPLE_CUDA_LIBS = $(CUDA_LIBS) # cuda.h couldn't be compiled with -pedantic flag EXAMPLE_CUDA_CFLAGS = EXAMPLE_CUDA_CPPFLAGS = $(CUDA_CPPFLAGS) -DHAVE_CUDA else -EXAMPLE_CUDA_LDFLAGS = +EXAMPLE_CUDA_LD_FLAGS = +EXAMPLE_CUDA_LIBS = EXAMPLE_CUDA_CFLAGS = $(CFLAGS_PEDANTIC) EXAMPLE_CUDA_CPPFLAGS = endif EXAMPLE_CCLD_FLAGS = -lucs -I$(includedir) -L$(libdir) -Wall -Werror -Wl,-rpath,$(libdir) \ - $(EXAMPLE_CUDA_LDFLAGS) $(EXAMPLE_CUDA_CPPFLAGS) + $(EXAMPLE_CUDA_LD_FLAGS) $(EXAMPLE_CUDA_LIBS) $(EXAMPLE_CUDA_CPPFLAGS) installcheck-local: @echo "INSTALLCHECK: Compiling examples with installed library" @@ -43,16 +45,18 @@ bin_PROGRAMS = \ ucp_hello_world_SOURCES = ucp_hello_world.c ucp_hello_world_CFLAGS = $(BASE_CFLAGS) $(EXAMPLE_CUDA_CFLAGS) ucp_hello_world_CPPFLAGS = $(BASE_CPPFLAGS) $(EXAMPLE_CUDA_CPPFLAGS) +ucp_hello_world_LDFLAGS = $(EXAMPLE_CUDA_LD_FLAGS) ucp_hello_world_LDADD = $(top_builddir)/src/ucs/libucs.la \ $(top_builddir)/src/ucp/libucp.la \ - $(EXAMPLE_CUDA_LDFLAGS) + $(EXAMPLE_CUDA_LIBS) uct_hello_world_SOURCES = uct_hello_world.c uct_hello_world_CFLAGS = $(BASE_CFLAGS) $(EXAMPLE_CUDA_CFLAGS) uct_hello_world_CPPFLAGS = $(BASE_CPPFLAGS) $(EXAMPLE_CUDA_CPPFLAGS) +uct_hello_world_LDFLAGS = $(EXAMPLE_CUDA_LD_FLAGS) uct_hello_world_LDADD = $(top_builddir)/src/ucs/libucs.la \ $(top_builddir)/src/uct/libuct.la \ - $(EXAMPLE_CUDA_LDFLAGS) + $(EXAMPLE_CUDA_LIBS) ucp_client_server_SOURCES = ucp_client_server.c ucp_client_server_CFLAGS = $(BASE_CFLAGS) $(CFLAGS_PEDANTIC) diff --git a/src/tools/perf/cuda/Makefile.am b/src/tools/perf/cuda/Makefile.am index aa6cb37065a..ecb7a33c0f7 100644 --- a/src/tools/perf/cuda/Makefile.am +++ b/src/tools/perf/cuda/Makefile.am @@ -10,6 +10,7 @@ module_LTLIBRARIES = libucx_perftest_cuda.la libucx_perftest_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucx_perftest_cuda_la_CFLAGS = $(BASE_CFLAGS) $(CUDA_CFLAGS) libucx_perftest_cuda_la_LDFLAGS = $(CUDA_LDFLAGS) -version-info $(SOVERSION) +libucx_perftest_cuda_la_LIBADD = $(CUDA_LIBS) libucx_perftest_cuda_la_SOURCES = cuda_alloc.c include $(top_srcdir)/config/module.am diff --git a/src/ucm/cuda/Makefile.am b/src/ucm/cuda/Makefile.am index 438960e2c28..03fd621f97a 100644 --- a/src/ucm/cuda/Makefile.am +++ b/src/ucm/cuda/Makefile.am @@ -9,7 +9,7 @@ if HAVE_CUDA module_LTLIBRARIES = libucm_cuda.la libucm_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucm_cuda_la_CFLAGS = $(BASE_CFLAGS) $(CUDA_CFLAGS) -libucm_cuda_la_LIBADD = ../libucm.la +libucm_cuda_la_LIBADD = ../libucm.la $(CUDA_LIBS) libucm_cuda_la_LDFLAGS = $(UCM_MODULE_LDFLAGS) \ $(patsubst %, -Xlinker %, $(CUDA_LDFLAGS)) \ -version-info $(SOVERSION) diff --git a/src/uct/cuda/Makefile.am b/src/uct/cuda/Makefile.am index 0992bb4d180..647092eda46 100644 --- a/src/uct/cuda/Makefile.am +++ b/src/uct/cuda/Makefile.am @@ -10,9 +10,10 @@ SUBDIRS = . gdr_copy module_LTLIBRARIES = libuct_cuda.la libuct_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libuct_cuda_la_CFLAGS = $(BASE_CFLAGS) $(CUDA_CFLAGS) -libuct_cuda_la_LIBADD = $(top_builddir)/src/ucs/libucs.la \ - $(top_builddir)/src/uct/libuct.la libuct_cuda_la_LDFLAGS = $(CUDA_LDFLAGS) -version-info $(SOVERSION) +libuct_cuda_la_LIBADD = $(top_builddir)/src/ucs/libucs.la \ + $(top_builddir)/src/uct/libuct.la \ + $(CUDA_LIBS) noinst_HEADERS = \ base/cuda_md.h \ diff --git a/src/uct/cuda/gdr_copy/Makefile.am b/src/uct/cuda/gdr_copy/Makefile.am index e551d5f5862..752fc984523 100644 --- a/src/uct/cuda/gdr_copy/Makefile.am +++ b/src/uct/cuda/gdr_copy/Makefile.am @@ -8,9 +8,11 @@ if HAVE_GDR_COPY module_LTLIBRARIES = libuct_cuda_gdrcopy.la libuct_cuda_gdrcopy_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) $(GDR_COPY_CPPFLAGS) libuct_cuda_gdrcopy_la_CFLAGS = $(BASE_CFLAGS) +libuct_cuda_gdrcopy_la_LDFLAGS = $(CUDA_LDFLAGS) $(GDR_COPY_LDFLAGS) \ + -version-info $(SOVERSION) libuct_cuda_gdrcopy_la_LIBADD = $(top_builddir)/src/ucs/libucs.la \ - $(top_builddir)/src/uct/cuda/libuct_cuda.la -libuct_cuda_gdrcopy_la_LDFLAGS = $(CUDA_LDFLAGS) $(GDR_COPY_LDFLAGS) -version-info $(SOVERSION) + $(top_builddir)/src/uct/cuda/libuct_cuda.la \ + $(CUDA_LIBS) noinst_HEADERS = \ gdr_copy_md.h \ diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am index 0a413f7af63..2ed94c8a16f 100644 --- a/test/apps/Makefile.am +++ b/test/apps/Makefile.am @@ -62,16 +62,18 @@ test_cuda_hook_dynamic_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) test_cuda_hook_dynamic_CFLAGS = $(BASE_CFLAGS) test_cuda_hook_dynamic_LDFLAGS = $(CUDA_LDFLAGS) test_cuda_hook_dynamic_LDADD = $(top_builddir)/src/ucp/libucp.la \ - $(top_builddir)/src/ucm/libucm.la + $(top_builddir)/src/ucm/libucm.la \ + $(CUDA_LIBS) if HAVE_CUDA_STATIC noinst_PROGRAMS += test_cuda_hook_static test_cuda_hook_static_SOURCES = test_cuda_hook.c test_cuda_hook_static_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) test_cuda_hook_static_CFLAGS = $(BASE_CFLAGS) -test_cuda_hook_static_LDFLAGS = $(CUDA_STATIC_LIB) -lcuda -lrt -ldl -lpthread +test_cuda_hook_static_LDFLAGS = $(CUDA_LDFLAGS) test_cuda_hook_static_LDADD = $(top_builddir)/src/ucp/libucp.la \ - $(top_builddir)/src/ucm/libucm.la + $(top_builddir)/src/ucm/libucm.la \ + $(CUDA_STATIC_LIBS) -lcuda -lrt -ldl -lpthread endif endif diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am index 0f3d24b30c8..202c7103391 100644 --- a/test/gtest/Makefile.am +++ b/test/gtest/Makefile.am @@ -206,7 +206,7 @@ endif if HAVE_TL_RC gtest_SOURCES += \ uct/ib/test_rc.cc -endif +endif if HAVE_TL_DC gtest_SOURCES += \ uct/ib/test_dc.cc @@ -222,8 +222,10 @@ gtest_SOURCES += \ ucm/cuda_hooks.cc gtest_CPPFLAGS += \ $(CUDA_CPPFLAGS) +gtest_LDFLAGS += \ + $(CUDA_LDFLAGS) gtest_LDADD += \ - $(CUDA_LDFLAGS) \ + $(CUDA_LIBS) \ $(top_builddir)/src/uct/cuda/libuct_cuda.la endif