Merge pull request #6089 from yosefe/topic/cuda-test-check-if-libcuda…

…rt-static-is CUDA/TEST: Check if libcudart_static is present for memory hook test
openucx · Jan 6, 2021 · 0d1ca83 · 0d1ca83
2 parents fd8f8b2 + c01b78f
commit 0d1ca83
Show file tree

Hide file tree

Showing 10 changed files with 76 additions and 33 deletions.
diff --git a/config/m4/cuda.m4 b/config/m4/cuda.m4
@@ -12,13 +12,19 @@ AS_IF([test "x$cuda_checked" != "xyes"],
                 [], [with_cuda=guess])
 
     AS_IF([test "x$with_cuda" = "xno"],
-        [cuda_happy=no],
+        [
+         cuda_happy=no
+         have_cuda_static=no
+        ],
         [
          save_CPPFLAGS="$CPPFLAGS"
          save_LDFLAGS="$LDFLAGS"
+         save_LIBS="$LIBS"
 
          CUDA_CPPFLAGS=""
          CUDA_LDFLAGS=""
+         CUDA_LIBS=""
+         CUDA_STATIC_LIBS=""
 
          AS_IF([test ! -z "$with_cuda" -a "x$with_cuda" != "xyes" -a "x$with_cuda" != "xguess"],
                [ucx_check_cuda_dir="$with_cuda"
@@ -40,18 +46,31 @@ AS_IF([test "x$cuda_checked" != "xyes"],
 
          # Check cuda libraries
          AS_IF([test "x$cuda_happy" = "xyes"],
-                [AC_CHECK_LIB([cuda], [cuDeviceGetUuid],
-                              [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcuda"], [cuda_happy="no"])])
+               [AC_CHECK_LIB([cuda], [cuDeviceGetUuid],
+                             [CUDA_LIBS="$CUDA_LIBS -lcuda"], [cuda_happy="no"])])
+         AS_IF([test "x$cuda_happy" = "xyes"],
+               [AC_CHECK_LIB([cudart], [cudaGetDeviceCount],
+                             [CUDA_LIBS="$CUDA_LIBS -lcudart"], [cuda_happy="no"])])
+
+         LDFLAGS="$save_LDFLAGS"
+
+         # Check for cuda static library
+         have_cuda_static="no"
          AS_IF([test "x$cuda_happy" = "xyes"],
-                [AC_CHECK_LIB([cudart], [cudaGetDeviceCount],
-                              [CUDA_LDFLAGS="$CUDA_LDFLAGS -lcudart"], [cuda_happy="no"])])
+               [AC_CHECK_LIB([cudart_static], [cudaGetDeviceCount],
+                             [CUDA_STATIC_LIBS="$CUDA_STATIC_LIBS -lcudart_static"
+                              have_cuda_static="yes"],
+                             [], [-ldl -lrt -lpthread])])
 
          CPPFLAGS="$save_CPPFLAGS"
          LDFLAGS="$save_LDFLAGS"
+         LIBS="$save_LIBS"
 
          AS_IF([test "x$cuda_happy" = "xyes"],
                [AC_SUBST([CUDA_CPPFLAGS], ["$CUDA_CPPFLAGS"])
                 AC_SUBST([CUDA_LDFLAGS], ["$CUDA_LDFLAGS"])
+                AC_SUBST([CUDA_LIBS], ["$CUDA_LIBS"])
+                AC_SUBST([CUDA_STATIC_LIBS], ["$CUDA_STATIC_LIBS"])
                 AC_DEFINE([HAVE_CUDA], 1, [Enable CUDA support])],
                [AS_IF([test "x$with_cuda" != "xguess"],
                       [AC_MSG_ERROR([CUDA support is requested but cuda packages cannot be found])],
@@ -61,6 +80,7 @@ AS_IF([test "x$cuda_checked" != "xyes"],
 
         cuda_checked=yes
         AM_CONDITIONAL([HAVE_CUDA], [test "x$cuda_happy" != xno])
+        AM_CONDITIONAL([HAVE_CUDA_STATIC], [test "X$have_cuda_static" = "Xyes"])
 
    ]) # "x$cuda_checked" != "xyes"
 

diff --git a/configure.ac b/configure.ac
@@ -185,6 +185,7 @@ AS_IF([test "x$with_docs_only" = xyes],
      AM_CONDITIONAL([HAVE_TL_CM], [false])
      AM_CONDITIONAL([HAVE_CRAY_UGNI], [false])
      AM_CONDITIONAL([HAVE_CUDA], [false])
+     AM_CONDITIONAL([HAVE_CUDA_STATIC], [false])
      AM_CONDITIONAL([HAVE_GDR_COPY], [false])
      AM_CONDITIONAL([HAVE_ROCM], [false])
      AM_CONDITIONAL([HAVE_HIP], [false])

diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh
@@ -1365,28 +1365,36 @@ test_malloc_hook() {
 
 	if [ "X$have_cuda" == "Xyes" ]
 	then
+		cuda_dynamic_exe=./test/apps/test_cuda_hook_dynamic
+		cuda_static_exe=./test/apps/test_cuda_hook_static
+
 		for mode in reloc bistro
 		do
 			export UCX_MEM_CUDA_HOOK_MODE=${mode}
 
 			# Run cuda memory hooks with dynamic link
-			./test/apps/test_cuda_hook_dynamic
+			${cuda_dynamic_exe}
 
-			# Run cuda memory hooks with static link
-			./test/apps/test_cuda_hook_static && status="pass" || status="fail"
-			[ ${mode} == "bistro" ] && exp_status="pass" || exp_status="fail"
-			if [ ${status} == ${exp_status} ]
+			# Run cuda memory hooks with static link, if exists. If the static
+			# library 'libcudart_static.a' is not present, static test will not
+			# be built.
+			if [ -x ${cuda_static_exe} ]
 			then
-				echo "Static link with cuda ${status}, as expected"
-			else
-				echo "Static link with cuda is expected to ${exp_status}, actual: ${status}"
-				exit 1
+				${cuda_static_exe} && status="pass" || status="fail"
+				[ ${mode} == "bistro" ] && exp_status="pass" || exp_status="fail"
+				if [ ${status} == ${exp_status} ]
+				then
+					echo "Static link with cuda ${status}, as expected"
+				else
+					echo "Static link with cuda is expected to ${exp_status}, actual: ${status}"
+					exit 1
+				fi
 			fi
 
 			# Test that driver API hooks work in both reloc and bistro modes,
 			# since we call them directly from the test
-			./test/apps/test_cuda_hook_dynamic -d
-			./test/apps/test_cuda_hook_static -d
+			${cuda_dynamic_exe} -d
+			[ -x ${cuda_static_exe} ] && ${cuda_static_exe} -d
 
 			unset UCX_MEM_CUDA_HOOK_MODE
 		done

diff --git a/examples/Makefile.am b/examples/Makefile.am
@@ -13,18 +13,20 @@ dist_examples_DATA =   \
 	ucp_client_server.c
 
 if HAVE_CUDA
-EXAMPLE_CUDA_LDFLAGS = $(CUDA_LDFLAGS)
+EXAMPLE_CUDA_LD_FLAGS = $(CUDA_LDFLAGS)
+EXAMPLE_CUDA_LIBS = $(CUDA_LIBS)
 # cuda.h couldn't be compiled with -pedantic flag
 EXAMPLE_CUDA_CFLAGS =
 EXAMPLE_CUDA_CPPFLAGS = $(CUDA_CPPFLAGS) -DHAVE_CUDA
 else
-EXAMPLE_CUDA_LDFLAGS =
+EXAMPLE_CUDA_LD_FLAGS =
+EXAMPLE_CUDA_LIBS =
 EXAMPLE_CUDA_CFLAGS = $(CFLAGS_PEDANTIC)
 EXAMPLE_CUDA_CPPFLAGS =
 endif
 
 EXAMPLE_CCLD_FLAGS = -lucs -I$(includedir) -L$(libdir) -Wall -Werror -Wl,-rpath,$(libdir) \
-                     $(EXAMPLE_CUDA_LDFLAGS) $(EXAMPLE_CUDA_CPPFLAGS)
+                     $(EXAMPLE_CUDA_LD_FLAGS) $(EXAMPLE_CUDA_LIBS) $(EXAMPLE_CUDA_CPPFLAGS)
 
 installcheck-local:
 	@echo "INSTALLCHECK: Compiling examples with installed library"
@@ -43,16 +45,18 @@ bin_PROGRAMS = \
 ucp_hello_world_SOURCES  = ucp_hello_world.c
 ucp_hello_world_CFLAGS   = $(BASE_CFLAGS) $(EXAMPLE_CUDA_CFLAGS)
 ucp_hello_world_CPPFLAGS = $(BASE_CPPFLAGS) $(EXAMPLE_CUDA_CPPFLAGS)
+ucp_hello_world_LDFLAGS  = $(EXAMPLE_CUDA_LD_FLAGS)
 ucp_hello_world_LDADD    = $(top_builddir)/src/ucs/libucs.la \
                            $(top_builddir)/src/ucp/libucp.la \
-                           $(EXAMPLE_CUDA_LDFLAGS)
+                           $(EXAMPLE_CUDA_LIBS)
 
 uct_hello_world_SOURCES  = uct_hello_world.c
 uct_hello_world_CFLAGS   = $(BASE_CFLAGS) $(EXAMPLE_CUDA_CFLAGS)
 uct_hello_world_CPPFLAGS = $(BASE_CPPFLAGS) $(EXAMPLE_CUDA_CPPFLAGS)
+uct_hello_world_LDFLAGS  = $(EXAMPLE_CUDA_LD_FLAGS)
 uct_hello_world_LDADD    = $(top_builddir)/src/ucs/libucs.la \
                            $(top_builddir)/src/uct/libuct.la \
-                           $(EXAMPLE_CUDA_LDFLAGS)
+                           $(EXAMPLE_CUDA_LIBS)
 
 ucp_client_server_SOURCES  = ucp_client_server.c
 ucp_client_server_CFLAGS   = $(BASE_CFLAGS) $(CFLAGS_PEDANTIC)

diff --git a/src/tools/perf/cuda/Makefile.am b/src/tools/perf/cuda/Makefile.am
@@ -10,6 +10,7 @@ module_LTLIBRARIES               = libucx_perftest_cuda.la
 libucx_perftest_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS)
 libucx_perftest_cuda_la_CFLAGS   = $(BASE_CFLAGS) $(CUDA_CFLAGS)
 libucx_perftest_cuda_la_LDFLAGS  = $(CUDA_LDFLAGS) -version-info $(SOVERSION)
+libucx_perftest_cuda_la_LIBADD   = $(CUDA_LIBS)
 libucx_perftest_cuda_la_SOURCES  = cuda_alloc.c
 
 include $(top_srcdir)/config/module.am

diff --git a/src/ucm/cuda/Makefile.am b/src/ucm/cuda/Makefile.am
@@ -9,7 +9,7 @@ if HAVE_CUDA
 module_LTLIBRARIES      = libucm_cuda.la
 libucm_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS)
 libucm_cuda_la_CFLAGS   = $(BASE_CFLAGS) $(CUDA_CFLAGS)
-libucm_cuda_la_LIBADD   = ../libucm.la
+libucm_cuda_la_LIBADD   = ../libucm.la $(CUDA_LIBS)
 libucm_cuda_la_LDFLAGS  = $(UCM_MODULE_LDFLAGS) \
                           $(patsubst %, -Xlinker %, $(CUDA_LDFLAGS)) \
                           -version-info $(SOVERSION)

diff --git a/src/uct/cuda/Makefile.am b/src/uct/cuda/Makefile.am
@@ -10,9 +10,10 @@ SUBDIRS = . gdr_copy
 module_LTLIBRARIES      = libuct_cuda.la
 libuct_cuda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS)
 libuct_cuda_la_CFLAGS   = $(BASE_CFLAGS) $(CUDA_CFLAGS)
-libuct_cuda_la_LIBADD   = $(top_builddir)/src/ucs/libucs.la \
-                          $(top_builddir)/src/uct/libuct.la
 libuct_cuda_la_LDFLAGS  = $(CUDA_LDFLAGS) -version-info $(SOVERSION)
+libuct_cuda_la_LIBADD   = $(top_builddir)/src/ucs/libucs.la \
+                          $(top_builddir)/src/uct/libuct.la \
+                          $(CUDA_LIBS)
 
 noinst_HEADERS = \
 	base/cuda_md.h \

diff --git a/src/uct/cuda/gdr_copy/Makefile.am b/src/uct/cuda/gdr_copy/Makefile.am
@@ -8,9 +8,11 @@ if HAVE_GDR_COPY
 module_LTLIBRARIES              = libuct_cuda_gdrcopy.la
 libuct_cuda_gdrcopy_la_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) $(GDR_COPY_CPPFLAGS)
 libuct_cuda_gdrcopy_la_CFLAGS   = $(BASE_CFLAGS)
+libuct_cuda_gdrcopy_la_LDFLAGS  = $(CUDA_LDFLAGS) $(GDR_COPY_LDFLAGS) \
+                                  -version-info $(SOVERSION)
 libuct_cuda_gdrcopy_la_LIBADD   = $(top_builddir)/src/ucs/libucs.la \
-                                  $(top_builddir)/src/uct/cuda/libuct_cuda.la
-libuct_cuda_gdrcopy_la_LDFLAGS  = $(CUDA_LDFLAGS) $(GDR_COPY_LDFLAGS) -version-info $(SOVERSION)
+                                  $(top_builddir)/src/uct/cuda/libuct_cuda.la \
+                                  $(CUDA_LIBS)
 
 noinst_HEADERS = \
 	gdr_copy_md.h \

diff --git a/test/apps/Makefile.am b/test/apps/Makefile.am
@@ -55,23 +55,27 @@ test_init_mt_CFLAGS   = $(BASE_CFLAGS) $(OPENMP_CFLAGS)
 test_init_mt_LDADD    = $(top_builddir)/src/ucp/libucp.la
 
 if HAVE_CUDA
-noinst_PROGRAMS                += test_cuda_hook_dynamic \
-                                  test_cuda_hook_static
+noinst_PROGRAMS                += test_cuda_hook_dynamic
 
 test_cuda_hook_dynamic_SOURCES  = test_cuda_hook.c
 test_cuda_hook_dynamic_CPPFLAGS = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS)
 test_cuda_hook_dynamic_CFLAGS   = $(BASE_CFLAGS)
 test_cuda_hook_dynamic_LDFLAGS  = $(CUDA_LDFLAGS)
 test_cuda_hook_dynamic_LDADD    = $(top_builddir)/src/ucp/libucp.la \
-                                  $(top_builddir)/src/ucm/libucm.la
+                                  $(top_builddir)/src/ucm/libucm.la \
+                                  $(CUDA_LIBS)
 
+if HAVE_CUDA_STATIC
 noinst_PROGRAMS                += test_cuda_hook_static
 test_cuda_hook_static_SOURCES   = test_cuda_hook.c
 test_cuda_hook_static_CPPFLAGS  = $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS)
 test_cuda_hook_static_CFLAGS    = $(BASE_CFLAGS)
-test_cuda_hook_static_LDFLAGS   = -lcudart_static -lcuda -lrt -ldl -lpthread
+test_cuda_hook_static_LDFLAGS   = $(CUDA_LDFLAGS)
 test_cuda_hook_static_LDADD     = $(top_builddir)/src/ucp/libucp.la \
-                                  $(top_builddir)/src/ucm/libucm.la
+                                  $(top_builddir)/src/ucm/libucm.la \
+                                  $(CUDA_STATIC_LIBS) -lcuda -lrt -ldl -lpthread
+endif
+
 endif
 
 if HAVE_TCMALLOC

diff --git a/test/gtest/Makefile.am b/test/gtest/Makefile.am
@@ -206,7 +206,7 @@ endif
 if HAVE_TL_RC
 gtest_SOURCES += \
 	uct/ib/test_rc.cc
-endif	
+endif
 if HAVE_TL_DC
 gtest_SOURCES += \
 	uct/ib/test_dc.cc
@@ -222,8 +222,10 @@ gtest_SOURCES += \
 	ucm/cuda_hooks.cc
 gtest_CPPFLAGS += \
 	$(CUDA_CPPFLAGS)
+gtest_LDFLAGS += \
+	$(CUDA_LDFLAGS)
 gtest_LDADD += \
-	$(CUDA_LDFLAGS) \
+	$(CUDA_LIBS) \
 	$(top_builddir)/src/uct/cuda/libuct_cuda.la
 endif