From 8801b17d03494e372a6f637d50286e97e30a7202 Mon Sep 17 00:00:00 2001 From: Elliot Ronaghan Date: Fri, 15 Sep 2023 13:26:28 -0400 Subject: [PATCH] Fix support for using a system hwloc and jemalloc Our chunk hooks (only used for some multi-locale configs) don't work with jemalloc 5.x (which switched to extent hooks) so don't try to enable them for 5.x. For system jemalloc and hwloc, provide compile/link args instead of assuming they are in a default search path. Lastly don't filter out include paths to jemalloc/hwloc in the qthreads makefile (qthreads users our allocator and hwloc so we need to provide it with paths to build.) This is motivated by wanting to use system hwloc/jemalloc for our homebrew formulae. Homebrew hwloc/jemalloc aren't available in default compiler search paths, so we need to use pkg-config to find them and ensure qthreads (which uses our hwloc and jemalloc) can find them too. Signed-off-by: Elliot Ronaghan --- runtime/src/mem/jemalloc/mem-jemalloc.c | 8 +++----- third-party/qthread/Makefile | 14 +++++++++++++- util/chplenv/chpl_hwloc.py | 5 +++-- util/chplenv/chpl_jemalloc.py | 5 +++-- util/chplenv/compile_link_args_utils.py | 2 +- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/runtime/src/mem/jemalloc/mem-jemalloc.c b/runtime/src/mem/jemalloc/mem-jemalloc.c index 6f8e7d89260c..d6f502452a4b 100644 --- a/runtime/src/mem/jemalloc/mem-jemalloc.c +++ b/runtime/src/mem/jemalloc/mem-jemalloc.c @@ -47,10 +47,8 @@ unsigned CHPL_JE_LG_ARENA; // Decide whether or not to try to use jemalloc's chunk hooks interface // jemalloc < 4.0 didn't support chunk_hooks_t // jemalloc 4.1 changed opt.nareas from size_t to unsigned -// .. so we use chunk hooks interface for jemalloc >= 4.1 -#if JEMALLOC_VERSION_MAJOR > 4 -#define USE_JE_CHUNK_HOOKS -#endif +// jemalloc 5.x migrated to an extent API +// .. so we use chunk hooks interface for jemalloc >= 4.1 and < 5.0 #if (JEMALLOC_VERSION_MAJOR == 4) && (JEMALLOC_VERSION_MINOR >= 1) #define USE_JE_CHUNK_HOOKS #endif @@ -302,7 +300,7 @@ static void replaceChunkHooks(void) { } } #else - chpl_internal_error("cannot init multi-locale heap: please rebuild with jemalloc >= 4.1"); + chpl_internal_error("cannot init multi-locale heap: please rebuild with jemalloc >= 4.1 and < 5.0"); #endif } diff --git a/third-party/qthread/Makefile b/third-party/qthread/Makefile index bf8145a1049a..e1198fc1fb3e 100644 --- a/third-party/qthread/Makefile +++ b/third-party/qthread/Makefile @@ -39,6 +39,8 @@ ifneq ($(CHPL_MAKE_HWLOC),none) endif # don't bother checking if we can link against hwloc CHPL_QTHREAD_CFG_OPTIONS += --disable-hwloc-configure-checks + else ifeq ($(CHPL_MAKE_HWLOC),system) + CHPL_QTHREAD_CFG_OPTIONS += --disable-hwloc-configure-checks endif endif @@ -73,6 +75,9 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y) # seems to cause Chapel programs built with the result to hang during # exit when shutting down Qthreads. # + # We also need to provide paths to system hwloc and jemalloc if they are + # enabled, since they might not be in default compiler search paths. + # # Note that we call compileline with CHPL_MAKE_COMM=none. Under # CHPL_MAKE_COMM=gasnet, compileline will fail if gasnet is not built, # so in order to avoid ordering/dependencies just ignore the comm @@ -83,6 +88,13 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y) # Throw COMPILELINE_STDERR_REDIRECT= if compileline failure is reported # and you want to see the stderr from that. # + ifeq ($(CHPL_MAKE_HWLOC),system) + SYSTEM_INCS_DEFS := $(SYSTEM_INCS_DEFS)\|hwloc + endif + ifeq ($(CHPL_MAKE_TARGET_JEMALLOC),system) + SYSTEM_INCS_DEFS := $(SYSTEM_INCS_DEFS)\|jemalloc + endif + COMPILELINE_STDERR_REDIRECT=2> /dev/null COMM_NONE_CHPLENV_CACHE := $(shell echo "$(CHPL_MAKE_CHPLENV_CACHE)" | sed 's/|CHPL_MAKE_COMM=[^|]*|/|CHPL_MAKE_COMM=none|/') INCS_DEFS := \ @@ -95,7 +107,7 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y) INCS_DEFS_PROCESSED := \ $(shell echo $(INCS_DEFS) \ | tr ' ' '\n' \ - | grep '^-DCHPL\|/runtime//*include\|/third-party/.*/install' \ + | grep '^-DCHPL\|/runtime//*include\|/third-party/.*/install$(SYSTEM_INCS_DEFS)' \ | grep -v '/third-party/qthread/install' \ | tr '\n' ' ' \ | sed 's/ $$//') diff --git a/util/chplenv/chpl_hwloc.py b/util/chplenv/chpl_hwloc.py index 38c24d710794..5f7ef44518d8 100755 --- a/util/chplenv/chpl_hwloc.py +++ b/util/chplenv/chpl_hwloc.py @@ -32,6 +32,8 @@ def get_compile_args(): if hwloc_val == 'bundled': ucp_val = get_uniq_cfg_path() return third_party_utils.get_bundled_compile_args('hwloc', ucp=ucp_val) + elif hwloc_val == 'system': + return third_party_utils.pkgconfig_get_system_compile_args('hwloc') return ([ ], [ ]) @@ -51,8 +53,7 @@ def get_link_args(): if exists and retcode != 0: err = "CHPL_HWLOC=system requires hwloc >= 2.1" error(err, ValueError) - - return ([ ], ['-lhwloc']) + return third_party_utils.pkgconfig_get_system_link_args('hwloc') return ([ ], [ ]) diff --git a/util/chplenv/chpl_jemalloc.py b/util/chplenv/chpl_jemalloc.py index b43bfafccb78..d1ac8fd30e52 100644 --- a/util/chplenv/chpl_jemalloc.py +++ b/util/chplenv/chpl_jemalloc.py @@ -94,6 +94,8 @@ def get_compile_args(flag): ucp_val = get_uniq_cfg_path(flag) return third_party_utils.get_bundled_compile_args('jemalloc', ucp=ucp_val) + elif jemalloc_val == 'system': + return third_party_utils.pkgconfig_get_system_compile_args('jemalloc') return ([ ], [ ]) # flag is host or target @@ -116,9 +118,8 @@ def get_link_args(flag): jemalloc_libs = run_command([jemalloc_config, '--libs']) morelibs += jemalloc_libs.split() return (libs, morelibs) - if jemalloc_val == 'system': - return ([ ], ['-ljemalloc']) + return third_party_utils.pkgconfig_get_system_link_args('jemalloc') return ([ ], [ ]) diff --git a/util/chplenv/compile_link_args_utils.py b/util/chplenv/compile_link_args_utils.py index fc9b325da3df..1435d1cbc60e 100644 --- a/util/chplenv/compile_link_args_utils.py +++ b/util/chplenv/compile_link_args_utils.py @@ -69,7 +69,7 @@ def get_runtime_includes_and_defines(): system.append("-isystem" + os.path.join(sdk_path, "hip", "include")) system.append("-isystem" + os.path.join(sdk_path, "hsa", "include")) - if mem == "jemalloc": + if mem == "jemalloc" and chpl_jemalloc.get('target') == 'bundled': # set -DCHPL_JEMALLOC_PREFIX=chpl_je_ # this is needed since it affects code inside of headers bundled.append("-DCHPL_JEMALLOC_PREFIX=chpl_je_")