Skip to content

Commit

Permalink
Make PyJulia usable in virtual environments
Browse files Browse the repository at this point in the history
  • Loading branch information
tkf committed Aug 29, 2018
1 parent 54d1116 commit 1825d34
Show file tree
Hide file tree
Showing 3 changed files with 235 additions and 27 deletions.
47 changes: 32 additions & 15 deletions julia/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
# this is python 3.3 specific
from types import ModuleType, FunctionType

from .find_libpython import find_libpython, normalize_path

#-----------------------------------------------------------------------------
# Classes and funtions
#-----------------------------------------------------------------------------
Expand Down Expand Up @@ -260,7 +262,11 @@ def determine_if_statically_linked():

JuliaInfo = namedtuple(
'JuliaInfo',
['JULIA_HOME', 'libjulia_path', 'image_file', 'pyprogramname'])
['JULIA_HOME', 'libjulia_path', 'image_file',
# Variables in PyCall/deps/deps.jl:
'pyprogramname', 'libpython'],
# PyCall/deps/deps.jl may not exist; The variables are then set to None:
defaults=[None, None])


def juliainfo(runtime='julia'):
Expand All @@ -283,28 +289,39 @@ def juliainfo(runtime='julia'):
if PyCall_depsfile !== nothing && isfile(PyCall_depsfile)
include(PyCall_depsfile)
println(pyprogramname)
println(libpython)
end
"""],
# Use the original environment variables to avoid a cryptic
# error "fake-julia/../lib/julia/sys.so: cannot open shared
# object file: No such file or directory":
env=_enviorn)
args = output.decode("utf-8").rstrip().split("\n")
if len(args) == 3:
args.append(None) # no pyprogramname set
return JuliaInfo(*args)


def is_same_path(a, b):
a = os.path.normpath(os.path.normcase(a))
b = os.path.normpath(os.path.normcase(b))
return a == b
def is_compatible_exe(jlinfo):
"""
Determine if Python used by PyCall.jl is compatible with this Python.
Current Python executable is considered compatible if it is dynamically
linked to libpython (usually the case in macOS and Windows) and
both of them are using identical libpython. If this function returns
`True`, PyJulia use the same precompilation cache of PyCall.jl used by
Julia itself.
Parameters
----------
jlinfo : JuliaInfo
A `JuliaInfo` object returned by `juliainfo` function.
"""
if jlinfo.libpython is None:
return False

if determine_if_statically_linked():
return False

def is_different_exe(pyprogramname, sys_executable):
if pyprogramname is None:
return True
return not is_same_path(pyprogramname, sys_executable)
return find_libpython() == normalize_path(jlinfo.libpython)


_julia_runtime = [False]
Expand Down Expand Up @@ -359,11 +376,10 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None,
runtime = jl_runtime_path
else:
runtime = 'julia'
JULIA_HOME, libjulia_path, image_file, depsjlexe = juliainfo(runtime)
jlinfo = juliainfo(runtime)
JULIA_HOME, libjulia_path, image_file, depsjlexe = jlinfo[:4]
self._debug("pyprogramname =", depsjlexe)
self._debug("sys.executable =", sys.executable)
exe_differs = is_different_exe(depsjlexe, sys.executable)
self._debug("exe_differs =", exe_differs)
self._debug("JULIA_HOME = %s, libjulia_path = %s" % (JULIA_HOME, libjulia_path))
if not os.path.exists(libjulia_path):
raise JuliaError("Julia library (\"libjulia\") not found! {}".format(libjulia_path))
Expand All @@ -381,7 +397,8 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None,
else:
jl_init_path = JULIA_HOME.encode("utf-8") # initialize with JULIA_HOME

use_separate_cache = exe_differs or determine_if_statically_linked()
use_separate_cache = not is_compatible_exe(jlinfo)
self._debug("use_separate_cache =", use_separate_cache)
if use_separate_cache:
PYCALL_JULIA_HOME = os.path.join(
os.path.dirname(os.path.realpath(__file__)),"fake-julia").replace("\\","\\\\")
Expand Down
199 changes: 199 additions & 0 deletions julia/find_libpython.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
#!/usr/bin/env python

"""
Locate libpython associated with this Python executable.
"""

from __future__ import print_function

from logging import getLogger
import ctypes.util
import os
import platform
import sys
import sysconfig

logger = getLogger("find_libpython")

SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") or ".so"


def library_name(name, suffix=SHLIB_SUFFIX,
is_windows=platform.system() == "Windows"):
"""
Convert a file basename `name` to a library name (no "lib" and ".so" etc.)
>>> library_name("libpython3.7m.so") # doctest: +SKIP
'python3.7m'
>>> library_name("libpython3.7m.so", suffix=".so", is_windows=False)
'python3.7m'
>>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False)
'python3.7m'
>>> library_name("python37.dll", suffix=".dll", is_windows=True)
'python37'
"""
if not is_windows:
name = name[len("lib"):]
if suffix and name.endswith(suffix):
name = name[:-len(suffix)]
return name


def append_truthy(list, item):
if item:
list.append(item)


def libpython_candidates(suffix=SHLIB_SUFFIX):
"""
Iterate over candidate paths of libpython.
Yields
------
path : str or None
Candidate path to libpython. The path may not be a fullpath
and may not exist.
"""
is_windows = platform.system() == "Windows"

# List candidates for libpython basenames
lib_basenames = []
append_truthy(lib_basenames, sysconfig.get_config_var("LDLIBRARY"))

LIBRARY = sysconfig.get_config_var("LIBRARY")
if LIBRARY:
lib_basenames.append(os.path.splitext(LIBRARY)[0] + suffix)

dlprefix = "" if is_windows else "lib"
sysdata = dict(
v=sys.version_info,
abiflags=(sysconfig.get_config_var("ABIFLAGS") or
sysconfig.get_config_var("abiflags") or ""),
)
lib_basenames.extend(dlprefix + p + suffix for p in [
"python{v.major}.{v.minor}{abiflags}".format(**sysdata),
"python{v.major}.{v.minor}".format(**sysdata),
"python{v.major}".format(**sysdata),
"python",
])

# List candidates for directories in which libpython may exist
lib_dirs = []
append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR"))

if is_windows:
lib_dirs.append(os.path.join(os.path.dirname(sys.executable)))
else:
lib_dirs.append(os.path.join(
os.path.dirname(os.path.dirname(sys.executable)),
"lib"))

# For macOS:
append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX"))

lib_dirs.append(sys.exec_prefix)
lib_dirs.append(os.path.join(sys.exec_prefix, "lib"))

for directory in lib_dirs:
for basename in lib_basenames:
yield os.path.join(directory, basename)

# In macOS and Windows, ctypes.util.find_library returns a full path:
for basename in lib_basenames:
yield ctypes.util.find_library(library_name(basename))


def normalize_path(path, suffix=SHLIB_SUFFIX):
"""
Normalize shared library `path` to a real path.
If `path` is not a full path, `None` is returned. If `path` does
not exists, append `SHLIB_SUFFIX` and check if it exists.
Finally, the path is canonicalized by following the symlinks.
Parameters
----------
path : str ot None
A candidate path to a shared library.
"""
if not path:
return None
if not os.path.isabs(path):
return None
if os.path.exists(path):
return os.path.realpath(path)
if os.path.exists(path + suffix):
return os.path.realpath(path + suffix)
return None


def finding_libpython():
"""
Iterate over existing libpython paths.
The first item is likely to be the best one. It may yield
duplicated paths.
Yields
------
path : str
Existing path to a libpython.
"""
for path in libpython_candidates():
logger.debug("Candidate: %s", path)
normalized = normalize_path(path)
logger.debug("Normalized: %s", normalized)
if normalized:
logger.debug("Found: %s", normalized)
yield normalized


def find_libpython():
"""
Return a path (`str`) to libpython or `None` if not found.
Parameters
----------
path : str or None
Existing path to the (supposedly) correct libpython.
"""
for path in finding_libpython():
return os.path.realpath(path)


def cli_find_libpython(verbose, list_all):
import logging
# Importing `logging` module here so that using `logging.debug`
# instead of `logger.debug` outside of this function becomes an
# error.

if verbose:
logging.basicConfig(level=logging.DEBUG)

if list_all:
for path in finding_libpython():
print(path)
return

path = find_libpython()
if path is None:
return 1
print(path, end="")


def main(args=None):
import argparse
parser = argparse.ArgumentParser(
description=__doc__)
parser.add_argument(
"--verbose", "-v", action="store_true",
help="Print debugging information.")
parser.add_argument(
"--list-all", action="store_true",
help="Print list of all paths found.")
ns = parser.parse_args(args)
parser.exit(cli_find_libpython(**vars(ns)))


if __name__ == "__main__":
main()
16 changes: 4 additions & 12 deletions test/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,9 @@
Unit tests which can be done without loading `libjulia`.
"""

import sys
from julia.find_libpython import finding_libpython

import pytest

from julia.core import is_different_exe


@pytest.mark.parametrize('pyprogramname, sys_executable, exe_differs', [
(sys.executable, sys.executable, False),
(None, sys.executable, True),
('/dev/null', sys.executable, True),
])
def test_is_different_exe(pyprogramname, sys_executable, exe_differs):
assert is_different_exe(pyprogramname, sys_executable) == exe_differs
def test_smoke_finding_libpython():
paths = list(finding_libpython())
assert set(map(type, paths)) == {str}

0 comments on commit 1825d34

Please sign in to comment.