Fix support for using a system hwloc and jemalloc

Our chunk hooks don't work with jemalloc 5.x (which switched to extent
hooks) so don't try to enable them for 5.x. For system jemalloc and
hwloc, provide compile/link args instead of assuming they are in a
default search path. Lastly don't filter out include paths to
jemalloc/hwloc in the qthreads makefile (qthreads users our allocator
and hwloc so we need to provide it with paths to build.)

Signed-off-by: Elliot Ronaghan <[email protected]>

runtime/src/mem/jemalloc/mem-jemalloc.c

@@ -47,10 +47,8 @@ unsigned CHPL_JE_LG_ARENA;
 // Decide whether or not to try to use jemalloc's chunk hooks interface
 //   jemalloc < 4.0 didn't support chunk_hooks_t
 //   jemalloc 4.1 changed opt.nareas from size_t to unsigned
-// .. so we use chunk hooks interface for jemalloc >= 4.1
-#if JEMALLOC_VERSION_MAJOR > 4
-#define USE_JE_CHUNK_HOOKS
-#endif
+//   jemalloc 5.x migrated to an extent API
+// .. so we use chunk hooks interface for jemalloc >= 4.1 and < 5.0
 #if (JEMALLOC_VERSION_MAJOR == 4) && (JEMALLOC_VERSION_MINOR >= 1)
 #define USE_JE_CHUNK_HOOKS
 #endif
@@ -302,7 +300,7 @@ static void replaceChunkHooks(void) {
     }
   }
 #else
-    chpl_internal_error("cannot init multi-locale heap: please rebuild with jemalloc >= 4.1");
+    chpl_internal_error("cannot init multi-locale heap: please rebuild with jemalloc >= 4.1 and < 5.0");
 #endif
 
 }

third-party/qthread/Makefile

@@ -39,6 +39,8 @@ ifneq ($(CHPL_MAKE_HWLOC),none)
     endif
     # don't bother checking if we can link against hwloc
     CHPL_QTHREAD_CFG_OPTIONS += --disable-hwloc-configure-checks
+  else ifeq ($(CHPL_MAKE_HWLOC),system)
+    CHPL_QTHREAD_CFG_OPTIONS += --disable-hwloc-configure-checks
   endif
 endif
 
@@ -73,6 +75,9 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y)
   # seems to cause Chapel programs built with the result to hang during
   # exit when shutting down Qthreads.
   #
+  # We also need to provide paths to system hwloc and jemalloc if they are
+  # enabled, since they might not be in default compiler search paths.
+  #
   # Note that we call compileline with CHPL_MAKE_COMM=none. Under
   # CHPL_MAKE_COMM=gasnet, compileline will fail if gasnet is not built,
   # so in order to avoid ordering/dependencies just ignore the comm
@@ -83,6 +88,13 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y)
   # Throw COMPILELINE_STDERR_REDIRECT= if compileline failure is reported
   # and you want to see the stderr from that.
   #
+  ifeq ($(CHPL_MAKE_HWLOC),system)
+	SYSTEM_INCS_DEFS := $(SYSTEM_INCS_DEFS)\|hwloc
+  endif
+  ifeq ($(CHPL_MAKE_TARGET_JEMALLOC),system)
+	SYSTEM_INCS_DEFS := $(SYSTEM_INCS_DEFS)\|jemalloc
+  endif
+
   COMPILELINE_STDERR_REDIRECT=2> /dev/null
   COMM_NONE_CHPLENV_CACHE := $(shell echo "$(CHPL_MAKE_CHPLENV_CACHE)" | sed 's/|CHPL_MAKE_COMM=[^|]*|/|CHPL_MAKE_COMM=none|/')
   INCS_DEFS := \
@@ -95,7 +107,7 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y)
     INCS_DEFS_PROCESSED := \
       $(shell echo $(INCS_DEFS) \
               | tr ' ' '\n' \
-              | grep '^-DCHPL\|/runtime//*include\|/third-party/.*/install' \
+              | grep '^-DCHPL\|/runtime//*include\|/third-party/.*/install$(SYSTEM_INCS_DEFS)' \
               | grep -v '/third-party/qthread/install' \
               | tr '\n' ' ' \
               | sed 's/ $$//')

util/chplenv/chpl_hwloc.py

@@ -32,6 +32,8 @@ def get_compile_args():
     if hwloc_val == 'bundled':
          ucp_val = get_uniq_cfg_path()
          return third_party_utils.get_bundled_compile_args('hwloc', ucp=ucp_val)
+    elif hwloc_val == 'system':
+         return third_party_utils.pkgconfig_get_system_compile_args('hwloc')
 
     return ([ ], [ ])
 
@@ -51,8 +53,7 @@ def get_link_args():
         if exists and retcode != 0:
           err = "CHPL_HWLOC=system requires hwloc >= 2.1"
           error(err, ValueError)
-
-        return ([ ], ['-lhwloc'])
+        return third_party_utils.pkgconfig_get_system_link_args('hwloc')
 
     return ([ ], [ ])
 

util/chplenv/chpl_jemalloc.py

@@ -94,6 +94,8 @@ def get_compile_args(flag):
         ucp_val = get_uniq_cfg_path(flag)
         return third_party_utils.get_bundled_compile_args('jemalloc',
                                                           ucp=ucp_val)
+    elif jemalloc_val == 'system':
+        return third_party_utils.pkgconfig_get_system_compile_args('jemalloc')
     return ([ ], [ ])
 
 # flag is host or target
@@ -116,9 +118,8 @@ def get_link_args(flag):
             jemalloc_libs = run_command([jemalloc_config, '--libs'])
             morelibs += jemalloc_libs.split()
         return (libs, morelibs)
-
     if jemalloc_val == 'system':
-        return ([ ], ['-ljemalloc'])
+        return third_party_utils.pkgconfig_get_system_link_args('jemalloc')
 
     return ([ ], [ ])
 

util/chplenv/compile_link_args_utils.py

@@ -69,7 +69,7 @@ def get_runtime_includes_and_defines():
             system.append("-isystem" + os.path.join(sdk_path, "hip", "include"))
             system.append("-isystem" + os.path.join(sdk_path, "hsa", "include"))
 
-    if mem == "jemalloc":
+    if mem == "jemalloc" and chpl_jemalloc.get() == 'bundled':
         # set -DCHPL_JEMALLOC_PREFIX=chpl_je_
         # this is needed since it affects code inside of headers
         bundled.append("-DCHPL_JEMALLOC_PREFIX=chpl_je_")