Patch build system to allow building with CHPL_HWLOC=system (#25145)

This PR patches some of our build scripts to work with
`CHPL_HWLOC=system`. By default, we will still continue to default to
`CHPL_HWLOC=bundled` with qthreads, but users can opt in to using a
system install by explicitly setting `CHPL_HWLOC=system`

This is a step in the right direction to allow us to use the preferred
Chapel configuration for Macs in homebrew.

Testing:
- [x] `start_test test/release/examples` with `CHPL_HWLOC=system` on Mac


Notes:
- Partially unblocks #24931
and #24655
- Used #23409 as a starting
point

[Reviewed by @jhh67]

third-party/qthread/Makefile

@@ -39,6 +39,9 @@ ifneq ($(CHPL_MAKE_HWLOC),none)
     endif
     # don't bother checking if we can link against hwloc
     CHPL_QTHREAD_CFG_OPTIONS += --disable-hwloc-configure-checks
+  else ifeq ($(CHPL_MAKE_HWLOC),system)
+		CHPL_QTHREAD_CFG_OPTIONS += --with-hwloc=$(shell $(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/chplenv/chpl_hwloc.py --prefix)
+    CHPL_QTHREAD_CFG_OPTIONS += --disable-hwloc-configure-checks
   endif
 endif
 
@@ -73,6 +76,9 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y)
   # seems to cause Chapel programs built with the result to hang during
   # exit when shutting down Qthreads.
   #
+	# We also need to provide paths to system hwloc if its enabled
+  # since it might not be in default compiler search paths.
+	#
   # Note that we call compileline with CHPL_MAKE_COMM=none. Under
   # CHPL_MAKE_COMM=gasnet, compileline will fail if gasnet is not built,
   # so in order to avoid ordering/dependencies just ignore the comm
@@ -83,6 +89,10 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y)
   # Throw COMPILELINE_STDERR_REDIRECT= if compileline failure is reported
   # and you want to see the stderr from that.
   #
+ 	ifeq ($(CHPL_MAKE_HWLOC),system)
+		SYSTEM_INCS_DEFS := $(SYSTEM_INCS_DEFS)\|hwloc
+  endif
+
   COMPILELINE_STDERR_REDIRECT=2> /dev/null
   COMM_NONE_CHPLENV_CACHE := $(shell echo "$(CHPL_MAKE_CHPLENV_CACHE)" | sed 's/|CHPL_MAKE_COMM=[^|]*|/|CHPL_MAKE_COMM=none|/')
   INCS_DEFS := \
@@ -95,7 +105,7 @@ ifeq ($(CFLAGS_NEEDS_RT_INCLUDES), y)
     INCS_DEFS_PROCESSED := \
       $(shell echo $(INCS_DEFS) \
               | tr ' ' '\n' \
-              | grep '^-DCHPL\|/runtime//*include\|/third-party/.*/install' \
+              | grep '^-DCHPL\|/runtime//*include\|/third-party/.*/install$(SYSTEM_INCS_DEFS)' \
               | grep -v '/third-party/qthread/install' \
               | tr '\n' ' ' \
               | sed 's/ $$//')

util/chplenv/chpl_hwloc.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 import sys
+import os
 
-import chpl_locale_model, chpl_tasks, overrides, third_party_utils
-from utils import error, memoize, warning, try_run_command
+import optparse
+import chpl_locale_model, chpl_tasks, chpl_platform, overrides, third_party_utils
+from utils import error, memoize, warning, try_run_command, run_command
 
 
 @memoize
@@ -30,8 +32,19 @@ def get_uniq_cfg_path():
 def get_compile_args():
     hwloc_val = get()
     if hwloc_val == 'bundled':
-         ucp_val = get_uniq_cfg_path()
-         return third_party_utils.get_bundled_compile_args('hwloc', ucp=ucp_val)
+        ucp_val = get_uniq_cfg_path()
+        return third_party_utils.get_bundled_compile_args('hwloc', ucp=ucp_val)
+    elif hwloc_val == 'system':
+        # try pkg-config
+        args = third_party_utils.pkgconfig_get_system_compile_args('hwloc')
+        if args != (None, None):
+            return args
+        # try homebrew
+        hwloc_prefix = chpl_platform.get_homebrew_prefix('hwloc')
+        if hwloc_prefix:
+            return ([], ['-I{0}'.format(os.path.join(hwloc_prefix, 'include'))])
+
+        error("Could not find a suitable hwloc installation.  Please install hwloc or set CHPL_HWLOC=bundled or CHPL_HWLOC=none.")
 
     return ([ ], [ ])
 
@@ -45,21 +58,70 @@ def get_link_args():
         return third_party_utils.pkgconfig_get_bundled_link_args(
                                           'hwloc', ucp=get_uniq_cfg_path())
     elif hwloc_val == 'system':
-        # Check that hwloc version is OK
-        exists, retcode, my_out, my_err = try_run_command(
-                            ['pkg-config', '--atleast-version=2.1', 'hwloc'])
-        if exists and retcode != 0:
-          error("CHPL_HWLOC=system requires hwloc >= 2.1", ValueError)
-
-        return ([ ], ['-lhwloc'])
+        if third_party_utils.has_pkgconfig():
+            # Check that hwloc version is OK
+            exists, retcode, my_out, my_err = try_run_command(
+                                ['pkg-config', '--atleast-version=2.1', 'hwloc'])
+            if exists and retcode != 0:
+                error("CHPL_HWLOC=system requires hwloc >= 2.1", ValueError)
+
+            return third_party_utils.pkgconfig_get_system_link_args('hwloc')
+        # try homebrew
+        hwloc_prefix = chpl_platform.get_homebrew_prefix('hwloc')
+        if hwloc_prefix:
+            # TODO: this should also check the version
+            return ([], ['-L{0}'.format(os.path.join(hwloc_prefix, 'lib')),
+                         '-lhwloc'])
+
+        error("Could not find a suitable hwloc installation. Please install hwloc or set CHPL_HWLOC=bundled or CHPL_HWLOC=none.")
 
     return ([ ], [ ])
 
+@memoize
+def get_prefix():
+    hwloc_val = get()
+    if hwloc_val == 'bundled':
+        ucp_val = get_uniq_cfg_path()
+        return third_party_utils.get_bundled_install_path('hwloc', ucp=ucp_val)
+    elif hwloc_val == 'system':
+        # try pkg-config
+        if third_party_utils.has_pkgconfig():
+            prefix = run_command(['pkg-config', '--variable', 'prefix', 'hwloc'])
+            if prefix:
+                return prefix.strip()
+        # try homebrew
+        hwloc_prefix = chpl_platform.get_homebrew_prefix('hwloc')
+        if hwloc_prefix:
+            return hwloc_prefix.strip()
+
+        error("Could not find a suitable hwloc installation.  Please install hwloc or set CHPL_HWLOC=bundled or CHPL_HWLOC=none.")
+
+    return ''
 
 def _main():
     hwloc_val = get()
-    sys.stdout.write("{0}\n".format(hwloc_val))
 
+    parser = optparse.OptionParser(usage='usage: %prog [--prefix] [--compile] [--link]')
+    parser.add_option('--prefix', dest='action',
+                      action='store_const',
+                      const='prefix', default='')
+    parser.add_option('--compile', dest='action',
+                      action='store_const',
+                      const='compile', default='')
+    parser.add_option('--link', dest='action',
+                      action='store_const',
+                      const='link', default='')
+
+    (options, args) = parser.parse_args()
+
+    if options.action == 'prefix':
+        sys.stdout.write("{0}\n".format(get_prefix()))
+    elif options.action == 'compile':
+        sys.stdout.write("{0}\n".format(get_compile_args()))
+    elif options.action == 'link':
+        sys.stdout.write("{0}\n".format(get_link_args()))
+    else:
+        sys.stdout.write("{0}\n".format(hwloc_val))
 
 if __name__ == '__main__':
     _main()

util/chplenv/chpl_platform.py

@@ -95,9 +95,12 @@ def is_arch_linux():
 # if running on a system with homebrew, return the homebrew prefix
 # if not, return None
 @memoize
-def get_homebrew_prefix():
+def get_homebrew_prefix(pkg=None):
     # Check to see if Homebrew is installed. If it is, return the prefix.
-    exists, retcode, my_out, my_err = try_run_command(['brew', '--prefix'])
+    cmd = ['brew', '--prefix']
+    if pkg is not None:
+        cmd.append(str(pkg))
+    exists, retcode, my_out, my_err = try_run_command(cmd)
     if exists and retcode == 0:
         # Make sure to include homebrew search path
         homebrew_prefix = my_out.strip()