From e617ad9c083090ffdc414d8d0d14b3b405aa1b0f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 31 Jul 2018 17:48:24 -0700 Subject: [PATCH] Make module loading lazy (#162) * Make module loading lazy * Catch less in JuliaModule.__getattr__ * Support star import * Special handling for Main module It adds an easier way to set variables in Julia's global namespace: >>> from julia import Main >>> Main.xs = [1, 2, 3] * Deprecate Julia.__getattr__ (and make it lazy) fixes #144 * Initialize Julia in JuliaModuleLoader if required This makes `from julia import ` works without the initial setup. Note that a custom setup can still be done by calling `julia.Julia` with appropriate arguments *before* trying to import Julia modules. closes #39, #79 * More interfaces to JuliaMainModule * Document the new API * Properly remove prefix * Fix JuliaModule.__all__; add .__dir__ * Cache original os.environ for tests * Simplify JuliaModule.__try_getattr * Add LegacyJulia to break infinite recursion * Don't import in isamodule Otherwise, trying to access undefined variables such as `julia.Base.__path__` produces a warning from Julia before `JuliaModule.__getattr__` raising `AttributeError`. * Remove core._orig_env; it can be done in test code * Use Enums as example sub-module as Base.REPL was pulled out from Base in Julia 0.7. * More import tests * Remove noop code * Fix wording --- README.md | 60 +++++++++-- julia/__init__.py | 2 +- julia/core.py | 249 ++++++++++++++++++++++++++----------------- test/_star_import.py | 1 + test/test_core.py | 70 ++++++++++++ 5 files changed, 278 insertions(+), 104 deletions(-) create mode 100644 test/_star_import.py diff --git a/README.md b/README.md index e4672c22..83e99397 100644 --- a/README.md +++ b/README.md @@ -58,24 +58,72 @@ If you run into problems using `pyjulia`, first check the version of `PyCall.jl` Usage ----- -To call Julia functions from python, first import the library + +`pyjulia` provides a high-level interface which assumes a "normal" +setup (e.g., `julia` is in your `PATH`) and a low-level interface +which can be used in a customized setup. + +### High-level interface + +To call a Julia function in a Julia module, import the Julia module +(say `Base`) with: + +```python +from julia import Base +``` + +and then call Julia functions in `Base` from python, e.g., ```python -import julia +Base.sind(90) ``` -then create a Julia object that makes a bridge to the Julia interpreter (assuming that `julia` is in your `PATH`) +Other variants of Python import syntax also work: ```python -j = julia.Julia() +import julia.Base +from julia.Base import LinAlg # import a submodule +from julia.Base import sin # import a function from a module ``` -You can then call Julia functions from python, e.g. +The global namespace of Julia's interpreter can be accessed via a +special module `julia.Main`: ```python -j.sind(90) +from julia import Main ``` +You can set names in this module to send Python values to Julia: + +```python +Main.xs = [1, 2, 3] +``` + +which allows it to be accessed directly from Julia code, e.g., it can +be evaluated at Julia side using Julia syntax: + +```python +Main.eval("sin.(xs)") +``` + +### Low-level interface + +If you need a custom setup for `pyjulia`, it must be done *before* +importing any Julia modules. For example, to use the Julia +interpreter at `PATH/TO/MY/CUSTOM/julia`, run: + +```python +from julia import Julia +j = julia.Julia(jl_runtime_path="PATH/TO/MY/CUSTOM/julia") +``` + +You can then use, e.g., + +```python +from julia import Base +``` + + How it works ------------ PyJulia loads the `libjulia` library and executes the statements therein. diff --git a/julia/__init__.py b/julia/__init__.py index 3a427ce3..fd4a2346 100644 --- a/julia/__init__.py +++ b/julia/__init__.py @@ -1 +1 @@ -from .core import Julia, JuliaError +from .core import JuliaError, LegacyJulia as Julia diff --git a/julia/core.py b/julia/core.py index b2982d04..bed7c3cd 100644 --- a/julia/core.py +++ b/julia/core.py @@ -14,7 +14,7 @@ #----------------------------------------------------------------------------- # Stdlib -from __future__ import print_function +from __future__ import print_function, absolute_import import ctypes import ctypes.util import os @@ -22,6 +22,7 @@ import keyword import subprocess import time +import warnings from ctypes import c_void_p as void_p from ctypes import c_char_p as char_p @@ -40,74 +41,135 @@ def iteritems(d): return iter(d.items()) else: iteritems = dict.iteritems - class JuliaError(Exception): pass +def remove_prefix(string, prefix): + return string[len(prefix):] if string.startswith(prefix) else string + + +def jl_name(name): + if name.endswith('_b'): + return name[:-2] + '!' + return name + + +def py_name(name): + if name.endswith('!'): + return name[:-1] + '_b' + return name + + class JuliaModule(ModuleType): - pass + def __init__(self, loader, *args, **kwargs): + super(JuliaModule, self).__init__(*args, **kwargs) + self._julia = loader.julia + self.__loader__ = loader + + @property + def __all__(self): + juliapath = remove_prefix(self.__name__, "julia.") + names = set(self._julia.eval("names({})".format(juliapath))) + names.discard(juliapath.rsplit('.', 1)[-1]) + return [py_name(n) for n in names if is_accessible_name(n)] + + def __dir__(self): + if python_version.major == 2: + names = set() + else: + names = set(super(JuliaModule, self).__dir__()) + names.update(self.__all__) + return list(names) + # Override __dir__ method so that completing member names work + # well in Python REPLs like IPython. + + def __getattr__(self, name): + try: + return self.__try_getattr(name) + except AttributeError: + if name.endswith("_b"): + try: + return self.__try_getattr(jl_name(name)) + except AttributeError: + pass + raise + + def __try_getattr(self, name): + jl_module = remove_prefix(self.__name__, "julia.") + jl_fullname = ".".join((jl_module, name)) + + # If `name` is a top-level module, don't import it as a + # submodule. Note that it handles the case that `name` is + # `Base` and `Core`. + is_toplevel = isdefined(self._julia, 'Main', name) + if not is_toplevel and isamodule(self._julia, jl_fullname): + # FIXME: submodules from other modules still hit this code + # path and they are imported as submodules. + return self.__loader__.load_module(".".join((self.__name__, name))) + + if isdefined(self._julia, jl_module, name): + return self._julia.eval(jl_fullname) + + raise AttributeError(name) + + +class JuliaMainModule(JuliaModule): + + def __setattr__(self, name, value): + if name.startswith('_'): + super(JuliaMainModule, self).__setattr__(name, value) + else: + juliapath = remove_prefix(self.__name__, "julia.") + setter = ''' + Main.PyCall.pyfunctionret( + (x) -> eval({}, :({} = $x)), + Any, + PyCall.PyAny) + '''.format(juliapath, jl_name(name)) + self._julia.eval(setter)(value) + + help = property(lambda self: self._julia.help) + eval = property(lambda self: self._julia.eval) + using = property(lambda self: self._julia.using) # add custom import behavior for the julia "module" class JuliaImporter(object): - def __init__(self, julia): - self.julia = julia # find_module was deprecated in v3.4 def find_module(self, fullname, path=None): - if path is None: - pass if fullname.startswith("julia."): - return JuliaModuleLoader(self.julia) + return JuliaModuleLoader() class JuliaModuleLoader(object): - def __init__(self, julia): - self.julia = julia + @property + def julia(self): + self.__class__.julia = julia = Julia() + return julia # load module was deprecated in v3.4 def load_module(self, fullname): - juliapath = fullname.lstrip("julia.") - if isamodule(self.julia, juliapath): - mod = sys.modules.setdefault(fullname, JuliaModule(fullname)) - mod.__loader__ = self - names = self.julia.eval("names({}, true, false)".format(juliapath)) - for name in names: - if (ismacro(name) or - isoperator(name) or - isprotected(name) or - notascii(name)): - continue - attrname = name - if name.endswith("!"): - attrname = name.replace("!", "_b") - if keyword.iskeyword(name): - attrname = "jl".join(name) - try: - module_path = ".".join((juliapath, name)) - module_obj = self.julia.eval(module_path) - is_module = self.julia.eval("isa({}, Module)" - .format(module_path)) - if is_module: - split_path = module_path.split(".") - is_base = split_path[-1] == "Base" - recur_module = split_path[-1] == split_path[-2] - if is_module and not is_base and not recur_module: - newpath = ".".join((fullname, name)) - module_obj = self.load_module(newpath) - setattr(mod, attrname, module_obj) - except Exception: - if isafunction(self.julia, name, mod_name=juliapath): - func = "{}.{}".format(juliapath, name) - setattr(mod, name, self.julia.eval(func)) - return mod + juliapath = remove_prefix(fullname, "julia.") + if juliapath == 'Main': + return sys.modules.setdefault(fullname, + JuliaMainModule(self, fullname)) elif isafunction(self.julia, juliapath): - return getattr(self.julia, juliapath) + return self.julia.eval(juliapath) + + try: + self.julia.eval("import {}".format(juliapath)) + except JuliaError: + pass else: - raise ImportError("{} not found".format(juliapath)) + if isamodule(self.julia, juliapath): + return sys.modules.setdefault(fullname, + JuliaModule(self, fullname)) + + raise ImportError("{} not found".format(juliapath)) def ismacro(name): @@ -137,19 +199,33 @@ def notascii(name): return True +def is_accessible_name(name): + """ + Check if a Julia variable `name` is (easily) accessible from Python. + + Return `True` if `name` can be safely converted to a Python + identifier using `py_name` function. For example, + + >>> is_accessible_name('A_mul_B!') + True + + Since it can be accessed as `A_mul_B_b` in Python. + """ + return not (ismacro(name) or + isoperator(name) or + isprotected(name) or + notascii(name)) + + +def isdefined(julia, parent, member): + return julia.eval("isdefined({}, :({}))".format(parent, member)) + + def isamodule(julia, julia_name): try: - ret = julia.eval("isa({}, Module)".format(julia_name)) - return ret - except: - # try explicitly importing it.. - try: - julia.eval("import {}".format(julia_name)) - ret = julia.eval("isa({}, Module)".format(julia_name)) - return ret - except: - pass - return False + return julia.eval("isa({}, Module)".format(julia_name)) + except JuliaError: + return False # assuming this is an `UndefVarError` def isafunction(julia, julia_name, mod_name=""): @@ -162,36 +238,6 @@ def isafunction(julia, julia_name, mod_name=""): return False -def module_functions(julia, module): - """Compute the function names in the julia module""" - bases = {} - names = julia.eval("names(%s)" % module) - for name in names: - if (ismacro(name) or - isoperator(name) or - isprotected(name) or - notascii(name)): - continue - try: - # skip undefined names - if not julia.eval("isdefined(:%s)" % name): - continue - # skip modules for now - if isamodule(julia, name): - continue - if name.startswith("_"): - continue - attr_name = name - if name.endswith("!"): - attr_name = name.replace("!", "_b") - if keyword.iskeyword(name): - attr_name = "jl".join(name) - julia_func = julia.eval(name) - bases[attr_name] = julia_func - except: - pass - return bases - def determine_if_statically_linked(): """Determines if this python executable is statically linked""" # Windows and OS X are generally always dynamically linked @@ -383,13 +429,8 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, # reloads. _julia_runtime[0] = self.api - self.add_module_functions("Base") - - sys.meta_path.append(JuliaImporter(self)) - - def add_module_functions(self, module): - for name, func in iteritems(module_functions(self, module)): - setattr(self, name, func) + self.sprint = self.eval('sprint') + self.showerror = self.eval('showerror') def _debug(self, msg): """ @@ -422,8 +463,8 @@ def check_exception(self, src=None): return # If, theoretically, an exception happens in early stage of - # self.add_module_functions("Base"), showerror and sprint as - # below does not work. Let's use jl_typeof_str in such case. + # self.__init__, showerror and sprint as below does not work. + # Let's use jl_typeof_str in such case. try: sprint = self.sprint showerror = self.showerror @@ -477,4 +518,18 @@ def _as_pyobj(self, res, src=None): def using(self, module): """Load module in Julia by calling the `using module` command""" self.eval("using %s" % module) - self.add_module_functions(module) + + +class LegacyJulia(Julia): + + def __getattr__(self, name): + from julia import Main + warnings.warn( + "Accessing `Julia().` to obtain Julia objects is" + " deprecated. Use `from julia import Main; Main.` or" + " `jl = Julia(); jl.eval('')`.", + DeprecationWarning) + return getattr(Main, name) + + +sys.meta_path.append(JuliaImporter()) diff --git a/test/_star_import.py b/test/_star_import.py new file mode 100644 index 00000000..2fd9ab8e --- /dev/null +++ b/test/_star_import.py @@ -0,0 +1 @@ +from julia.Base.Enums import * diff --git a/test/test_core.py b/test/test_core.py index 3ace81c5..4391901d 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -1,14 +1,20 @@ +from __future__ import print_function + import array import math +import subprocess import unittest +from types import ModuleType from julia import Julia, JuliaError +from julia.core import jl_name, py_name import sys import os python_version = sys.version_info +orig_env = os.environ.copy() julia = Julia(jl_runtime_path=os.getenv("JULIA_EXE"), debug=True) class JuliaTest(unittest.TestCase): @@ -62,9 +68,73 @@ def test_import_julia_functions(self): else: pass + def test_import_julia_module_existing_function(self): + from julia import Base + assert Base.mod(2, 2) == 0 + + def test_from_import_existing_julia_function(self): + from julia.Base import divrem + assert divrem(7, 3) == (2, 1) + + def test_import_julia_module_non_existing_name(self): + from julia import Base + try: + Base.spamspamspam + self.fail('No AttributeError') + except AttributeError: + pass + + def test_from_import_non_existing_julia_name(self): + try: + from Base import spamspamspam + except ImportError: + pass + else: + assert not spamspamspam + + def test_julia_module_bang(self): + from julia import Base + xs = [1, 2, 3] + ys = Base.scale_b(xs[:], 2) + assert all(x * 2 == y for x, y in zip(xs, ys)) + + def test_import_julia_submodule(self): + from julia.Base import Enums + assert isinstance(Enums, ModuleType) + + def test_star_import_julia_module(self): + from . import _star_import + _star_import.Enum + + def test_main_module(self): + from julia import Main + Main.x = x = 123456 + assert julia.eval('x') == x + + def test_module_all(self): + from julia import Base + assert 'resize_b' in Base.__all__ + + def test_module_dir(self): + from julia import Base + assert 'resize_b' in dir(Base) + + def test_import_without_setup(self): + command = [sys.executable, '-c', 'from julia import Base'] + print('Executing:', *command) + subprocess.check_call(command, env=orig_env) + #TODO: this causes a segfault """ def test_import_julia_modules(self): import julia.PyCall as pycall self.assertEquals(6, pycall.pyeval('2 * 3')) """ + + def test_jlpy_identity(self): + for name in ['normal', 'resize!']: + self.assertEqual(jl_name(py_name(name)), name) + + def test_pyjl_identity(self): + for name in ['normal', 'resize_b']: + self.assertEqual(py_name(jl_name(name)), name)