DAOS-16365 client: intercept dlsym() and zeInit() to avoid nested call (

#14932) (#15446)

libfabric loads libze_loader.so which calls zeInit(). We observed deadlock due to nested calls when daos_init() is called inside zeInit(). We intercept dlsym() and zeInit() to avoid calling daos_init() inside zeInit(). dlsym(RTLD_NEXT, ) checks returning address to determine caller's module. To maintain expected behavior of dlsym(RTLD_NEXT, ) with our interception, new_dlsym() is implemented with assembly code to use jmp instruction instead of call. dlsym() has been moved from libdl.so to libc.so since version 2.34.

Signed-off-by: Lei Huang <[email protected]>

src/client/dfuse/pil4dfs/hook.c

@@ -42,6 +42,7 @@
 #include <linux/limits.h>
 #include <capstone/capstone.h>
 #include <gurt/common.h>
+#include <gnu/libc-version.h>
 
 #include "hook.h"
 #include "hook_int.h"
@@ -89,10 +90,15 @@ static uint64_t lib_base_addr[MAX_NUM_LIB];
 /* List of names of loaded libraries */
 static char   **lib_name_list;
 
+/* libc version number in current process. e.g., 2.28 */
+static float    libc_version;
+static char    *libc_version_str;
+
 /* end   to compile list of memory blocks in /proc/pid/maps */
 
 static char    *path_ld;
 static char    *path_libc;
+static char    *path_libdl;
 static char    *path_libpthread;
 /* This holds the path of libpil4dfs.so. It is needed when we want to
  * force child processes append libpil4dfs.so to env LD_PRELOAD. */
@@ -212,7 +218,7 @@ determine_lib_path(void)
 {
 	int   path_offset   = 0, read_size, i, rc;
 	char *read_buff_map = NULL;
-	char *pos, *start, *end, lib_ver_str[32] = "", *lib_dir_str = NULL;
+	char *pos, *start, *end, *lib_dir_str = NULL;
 
 	read_size = read_map_file(&read_buff_map);
 
@@ -289,19 +295,17 @@ determine_lib_path(void)
 		goto err;
 	path_libc[end - start] = 0;
 
-	pos = strstr(path_libc, "libc-2.");
-	if (pos) {
-		/* containing version in name. example, 2.17 */
-		memcpy(lib_ver_str, pos + 5, 4);
-		lib_ver_str[4] = 0;
+	if (libc_version_str == NULL) {
+		libc_version_str = (char *)gnu_get_libc_version();
+		if (libc_version_str == NULL) {
+			DS_ERROR(errno, "Failed to determine libc version");
+			goto err;
+		}
+		libc_version = atof(libc_version_str);
 	}
 
-	if (lib_ver_str[0]) {
-		/* with version in name */
-		rc = asprintf(&path_libpthread, "%s/libpthread-%s.so", lib_dir_str, lib_ver_str);
-	} else {
-		rc = asprintf(&path_libpthread, "%s/libpthread.so.0", lib_dir_str);
-	}
+	/* with version in name */
+	rc = asprintf(&path_libpthread, "%s/libpthread-%s.so", lib_dir_str, libc_version_str);
 	if (rc < 0) {
 		DS_ERROR(ENOMEM, "Failed to allocate memory for path_libpthread");
 		goto err_1;
@@ -311,7 +315,18 @@ determine_lib_path(void)
 		path_libpthread = NULL;
 		DS_ERROR(ENAMETOOLONG, "path_libpthread is too long");
 		goto err_1;
-	}	
+	}
+	rc = asprintf(&path_libdl, "%s/libdl-%s.so", lib_dir_str, libc_version_str);
+	if (rc < 0) {
+		DS_ERROR(ENOMEM, "Failed to allocate memory for path_libdl");
+		goto err_1;
+	}
+	if (rc >= PATH_MAX) {
+		free(path_libdl);
+		path_libdl = NULL;
+		DS_ERROR(ENAMETOOLONG, "path_libdl is too long");
+		goto err_1;
+	}
 	D_FREE(lib_dir_str);
 
 	pos = strstr(read_buff_map, "libpil4dfs.so");
@@ -348,6 +363,11 @@ query_pil4dfs_path(void)
 	return path_libpil4dfs;
 }
 
+float
+query_libc_version(void)
+{
+	return libc_version;
+}
 
 /*
  * query_func_addr - Determine the addresses and code sizes of functions in func_name_list[].
@@ -754,6 +774,7 @@ free_memory_in_hook(void)
 	D_FREE(path_ld);
 	D_FREE(path_libc);
 	D_FREE(module_list);
+	free(path_libdl);
 	free(path_libpthread);
 
 	if (lib_name_list) {
@@ -1034,6 +1055,8 @@ register_a_hook(const char *module_name, const char *func_name, const void *new_
 		module_name_local = path_ld;
 	else if (strncmp(module_name, "libc", 5) == 0)
 		module_name_local = path_libc;
+	else if (strncmp(module_name, "libdl", 6) == 0)
+		module_name_local = path_libdl;
 	else if (strncmp(module_name, "libpthread", 11) == 0)
 		module_name_local = path_libpthread;
 	else

src/client/dfuse/pil4dfs/hook.h

@@ -60,4 +60,10 @@ free_memory_in_hook(void);
 char *
 query_pil4dfs_path(void);
 
+/**
+ * return glibc version in current process
+ */
+float
+query_libc_version(void);
+
 #endif