diff --git a/julia/core.py b/julia/core.py index 27c82779..50f03b53 100644 --- a/julia/core.py +++ b/julia/core.py @@ -32,6 +32,8 @@ # this is python 3.3 specific from types import ModuleType, FunctionType +from .find_libpython import find_libpython, normalize_path + #----------------------------------------------------------------------------- # Classes and funtions #----------------------------------------------------------------------------- @@ -260,7 +262,11 @@ def determine_if_statically_linked(): JuliaInfo = namedtuple( 'JuliaInfo', - ['JULIA_HOME', 'libjulia_path', 'image_file', 'pyprogramname']) + ['JULIA_HOME', 'libjulia_path', 'image_file', + # Variables in PyCall/deps/deps.jl: + 'pyprogramname', 'libpython'], + # PyCall/deps/deps.jl may not exist; The variables are then set to None: + defaults=[None, None]) def juliainfo(runtime='julia'): @@ -283,6 +289,7 @@ def juliainfo(runtime='julia'): if PyCall_depsfile !== nothing && isfile(PyCall_depsfile) include(PyCall_depsfile) println(pyprogramname) + println(libpython) end """], # Use the original environment variables to avoid a cryptic @@ -290,21 +297,31 @@ def juliainfo(runtime='julia'): # object file: No such file or directory": env=_enviorn) args = output.decode("utf-8").rstrip().split("\n") - if len(args) == 3: - args.append(None) # no pyprogramname set return JuliaInfo(*args) -def is_same_path(a, b): - a = os.path.normpath(os.path.normcase(a)) - b = os.path.normpath(os.path.normcase(b)) - return a == b +def is_compatible_exe(jlinfo): + """ + Determine if Python used by PyCall.jl is compatible with this Python. + + Current Python executable is considered compatible if it is dynamically + linked to libpython (usually the case in macOS and Windows) and + both of them are using identical libpython. If this function returns + `True`, PyJulia use the same precompilation cache of PyCall.jl used by + Julia itself. + + Parameters + ---------- + jlinfo : JuliaInfo + A `JuliaInfo` object returned by `juliainfo` function. + """ + if jlinfo.libpython is None: + return False + if determine_if_statically_linked(): + return False -def is_different_exe(pyprogramname, sys_executable): - if pyprogramname is None: - return True - return not is_same_path(pyprogramname, sys_executable) + return find_libpython() == normalize_path(jlinfo.libpython) _julia_runtime = [False] @@ -359,11 +376,10 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, runtime = jl_runtime_path else: runtime = 'julia' - JULIA_HOME, libjulia_path, image_file, depsjlexe = juliainfo(runtime) + jlinfo = juliainfo(runtime) + JULIA_HOME, libjulia_path, image_file, depsjlexe = jlinfo[:4] self._debug("pyprogramname =", depsjlexe) self._debug("sys.executable =", sys.executable) - exe_differs = is_different_exe(depsjlexe, sys.executable) - self._debug("exe_differs =", exe_differs) self._debug("JULIA_HOME = %s, libjulia_path = %s" % (JULIA_HOME, libjulia_path)) if not os.path.exists(libjulia_path): raise JuliaError("Julia library (\"libjulia\") not found! {}".format(libjulia_path)) @@ -381,7 +397,8 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, else: jl_init_path = JULIA_HOME.encode("utf-8") # initialize with JULIA_HOME - use_separate_cache = exe_differs or determine_if_statically_linked() + use_separate_cache = not is_compatible_exe(jlinfo) + self._debug("use_separate_cache =", use_separate_cache) if use_separate_cache: PYCALL_JULIA_HOME = os.path.join( os.path.dirname(os.path.realpath(__file__)),"fake-julia").replace("\\","\\\\") diff --git a/julia/find_libpython.py b/julia/find_libpython.py new file mode 100755 index 00000000..686aa736 --- /dev/null +++ b/julia/find_libpython.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +""" +Locate libpython associated with this Python executable. +""" + +from __future__ import print_function + +from logging import getLogger +import ctypes.util +import os +import platform +import sys +import sysconfig + +logger = getLogger("find_libpython") + +SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") or ".so" + + +def library_name(name, suffix=SHLIB_SUFFIX, + is_windows=platform.system() == "Windows"): + """ + Convert a file basename `name` to a library name (no "lib" and ".so" etc.) + + >>> library_name("libpython3.7m.so") # doctest: +SKIP + 'python3.7m' + >>> library_name("libpython3.7m.so", suffix=".so", is_windows=False) + 'python3.7m' + >>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False) + 'python3.7m' + >>> library_name("python37.dll", suffix=".dll", is_windows=True) + 'python37' + """ + if not is_windows: + name = name[len("lib"):] + if suffix and name.endswith(suffix): + name = name[:-len(suffix)] + return name + + +def append_truthy(list, item): + if item: + list.append(item) + + +def libpython_candidates(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate paths of libpython. + + Yields + ------ + path : str or None + Candidate path to libpython. The path may not be a fullpath + and may not exist. + """ + is_windows = platform.system() == "Windows" + + # List candidates for libpython basenames + lib_basenames = [] + append_truthy(lib_basenames, sysconfig.get_config_var("LDLIBRARY")) + + LIBRARY = sysconfig.get_config_var("LIBRARY") + if LIBRARY: + lib_basenames.append(os.path.splitext(LIBRARY)[0] + suffix) + + dlprefix = "" if is_windows else "lib" + sysdata = dict( + v=sys.version_info, + abiflags=(sysconfig.get_config_var("ABIFLAGS") or + sysconfig.get_config_var("abiflags") or ""), + ) + lib_basenames.extend(dlprefix + p + suffix for p in [ + "python{v.major}.{v.minor}{abiflags}".format(**sysdata), + "python{v.major}.{v.minor}".format(**sysdata), + "python{v.major}".format(**sysdata), + "python", + ]) + + # List candidates for directories in which libpython may exist + lib_dirs = [] + append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR")) + + if is_windows: + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + else: + lib_dirs.append(os.path.join( + os.path.dirname(os.path.dirname(sys.executable)), + "lib")) + + # For macOS: + append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")) + + lib_dirs.append(sys.exec_prefix) + lib_dirs.append(os.path.join(sys.exec_prefix, "lib")) + + for directory in lib_dirs: + for basename in lib_basenames: + yield os.path.join(directory, basename) + + # In macOS and Windows, ctypes.util.find_library returns a full path: + for basename in lib_basenames: + yield ctypes.util.find_library(library_name(basename)) + + +def normalize_path(path, suffix=SHLIB_SUFFIX): + """ + Normalize shared library `path` to a real path. + + If `path` is not a full path, `None` is returned. If `path` does + not exists, append `SHLIB_SUFFIX` and check if it exists. + Finally, the path is canonicalized by following the symlinks. + + Parameters + ---------- + path : str ot None + A candidate path to a shared library. + """ + if not path: + return None + if not os.path.isabs(path): + return None + if os.path.exists(path): + return os.path.realpath(path) + if os.path.exists(path + suffix): + return os.path.realpath(path + suffix) + return None + + +def finding_libpython(): + """ + Iterate over existing libpython paths. + + The first item is likely to be the best one. It may yield + duplicated paths. + + Yields + ------ + path : str + Existing path to a libpython. + """ + for path in libpython_candidates(): + logger.debug("Candidate: %s", path) + normalized = normalize_path(path) + logger.debug("Normalized: %s", normalized) + if normalized: + logger.debug("Found: %s", normalized) + yield normalized + + +def find_libpython(): + """ + Return a path (`str`) to libpython or `None` if not found. + + Parameters + ---------- + path : str or None + Existing path to the (supposedly) correct libpython. + """ + for path in finding_libpython(): + return os.path.realpath(path) + + +def cli_find_libpython(verbose, list_all): + import logging + # Importing `logging` module here so that using `logging.debug` + # instead of `logger.debug` outside of this function becomes an + # error. + + if verbose: + logging.basicConfig(level=logging.DEBUG) + + if list_all: + for path in finding_libpython(): + print(path) + return + + path = find_libpython() + if path is None: + return 1 + print(path, end="") + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Print debugging information.") + parser.add_argument( + "--list-all", action="store_true", + help="Print list of all paths found.") + ns = parser.parse_args(args) + parser.exit(cli_find_libpython(**vars(ns))) + + +if __name__ == "__main__": + main() diff --git a/test/test_utils.py b/test/test_utils.py index a86d146b..a8c15c88 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -2,17 +2,9 @@ Unit tests which can be done without loading `libjulia`. """ -import sys +from julia.find_libpython import finding_libpython -import pytest -from julia.core import is_different_exe - - -@pytest.mark.parametrize('pyprogramname, sys_executable, exe_differs', [ - (sys.executable, sys.executable, False), - (None, sys.executable, True), - ('/dev/null', sys.executable, True), -]) -def test_is_different_exe(pyprogramname, sys_executable, exe_differs): - assert is_different_exe(pyprogramname, sys_executable) == exe_differs +def test_smoke_finding_libpython(): + paths = list(finding_libpython()) + assert set(map(type, paths)) == {str}