From 54c22d07ef261c42a8b2164b05da6cf22fdd355a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 13 Nov 2024 14:24:22 -0800 Subject: [PATCH 1/3] prefer wheel-provided libcudf.so in load_library(), use RTLD_LOCAL --- python/libcudf/libcudf/load.py | 70 ++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index bf27ecfa7f5..dbce34dedb8 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -16,8 +16,36 @@ import ctypes import os +# RTLD_LOCAL is here for safety... using it loads symbols into the library-specific +# table maintained by the loader, but not into the global namespace where they +# may conflict with symbols from other loaded DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + out = None + for lib_dir in ("lib", "lib64"): + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), lib_dir, soname) + ): + out = ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + break + return out + def load_library(): + """Dynamically load libcudf.so and its dependencies""" try: # libkvikio must be loaded before libcudf because libcudf references its symbols import libkvikio @@ -29,28 +57,28 @@ def load_library(): # we assume the library is discoverable on system paths. pass - # Dynamically load libcudf.so. Prefer a system library if one is present to - # avoid clobbering symbols that other packages might expect, but if no - # other library is present use the one in the wheel. + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUDF_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + soname = "libcudf.so" libcudf_lib = None - try: - libcudf_lib = ctypes.CDLL("libcudf.so", ctypes.RTLD_GLOBAL) - except OSError: - # If neither of these directories contain the library, we assume we are in an - # environment where the C++ library is already installed somewhere else and the - # CMake build of the libcudf Python package was a no-op. - # - # Note that this approach won't work for real editable installs of the libcudf package. - # scikit-build-core has limited support for importlib.resources so there isn't a clean - # way to support that case yet. - for lib_dir in ("lib", "lib64"): - if os.path.isfile( - lib := os.path.join( - os.path.dirname(__file__), lib_dir, "libcudf.so" - ) - ): - libcudf_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) - break + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcudf_lib = _load_system_installation(soname) + except OSError: + libcudf_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt before + # packaging the wheel), look for a system installation. + libcudf_lib = _load_wheel_installation(soname) + if libcudf_lib is None: + libcudf_lib = _load_system_installation(soname) # The caller almost never needs to do anything with this library, but no # harm in offering the option since this object at least provides a handle From 17598fb87fc1ec0d1c815e7b3f7fded565e44ab4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 14 Nov 2024 09:32:56 -0600 Subject: [PATCH 2/3] just search lib64, update comment --- python/libcudf/libcudf/load.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index dbce34dedb8..82c789d6728 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -16,9 +16,12 @@ import ctypes import os -# RTLD_LOCAL is here for safety... using it loads symbols into the library-specific -# table maintained by the loader, but not into the global namespace where they -# may conflict with symbols from other loaded DSOs. +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libcuspatial that could conflict with symbols from other DSOs. PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL @@ -34,14 +37,11 @@ def _load_wheel_installation(soname: str): Returns ``None`` if the library cannot be loaded. """ - out = None - for lib_dir in ("lib", "lib64"): - if os.path.isfile( - lib := os.path.join(os.path.dirname(__file__), lib_dir, soname) - ): - out = ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) - break - return out + if os.path.isfile( + lib := os.path.join(os.path.dirname(__file__), "lib64", soname) + ): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None def load_library(): From a62dfd2e9148df0e48f8c472fc79c9cc556af864 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 14 Nov 2024 15:07:25 -0600 Subject: [PATCH 3/3] Update python/libcudf/libcudf/load.py --- python/libcudf/libcudf/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index 82c789d6728..a91fbb7aecf 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -21,7 +21,7 @@ # namespace. This allows libraries that express a dependency on # this library to be loaded later and successfully satisfy this dependency # without polluting the global symbol table with symbols from -# libcuspatial that could conflict with symbols from other DSOs. +# libcudf that could conflict with symbols from other DSOs. PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL