determine libpython at Pkg.build time, not at runtime (for #167) (fixes

#91)

.gitignore

@@ -0,0 +1 @@
+deps/deps.jl

.travis.yml

@@ -1,6 +1,5 @@
 language: python
 python:
-    - 2.6
     - 2.7
     - 3.2
     - 3.3
@@ -22,7 +21,7 @@ before_install:
   - git config --global user.email "[email protected]"
   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
 script:
-  - julia -e 'versioninfo(); Pkg.init(); Pkg.clone(pwd())'
+  - julia -e 'versioninfo(); Pkg.init(); Pkg.clone(pwd()); Pkg.build("PyCall")'
   - if [ $JULIAVERSION = "julianightlies" ]; then julia --code-coverage test/runtests.jl; fi
   - if [ $JULIAVERSION = "juliareleases" ]; then julia test/runtests.jl; fi
 after_success:

README.md

@@ -17,7 +17,7 @@ call Python functions from the Julia language with type conversions.
 ## Installation
 
 Within Julia, just use the package manager to run `Pkg.add("PyCall")` to
-install the files.  Julia 0.3 or later is required.
+install the files.  Julia 0.3 or later and Python 2.7 or later are required.
 
 The latest development version of PyCall is avalable from
 <https://github.com/stevengj/PyCall.jl>.  If you want to switch to
@@ -26,7 +26,32 @@ this after installing the package, run `Pkg.checkout("PyCall")`.
 You must, of course, have Python and its `libpython` shared-library
 installed, and a `python` executable must be in your `PATH` (or be
 specified manually as described below).  Usually, the necessary
-libraries are installed along with Python, but [pyenv on MacOS](https://github.com/stevengj/PyCall.jl/issues/122) requires you to install it with `env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install 3.4.3`.
+libraries are installed along with Python, but [pyenv on MacOS](https://github.com/stevengj/PyCall.jl/issues/122) requires you to install it with `env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install 3.4.3`.  The Enthought Canopy Python distribution is currently [not supported](https://github.com/stevengj/PyCall.jl/issues/42).
+
+As a general rule, we tend to recommend the [Anaconda Python
+distribution](https://store.continuum.io/cshop/anaconda/) on MacOS and
+Windows in order to minimize headaches.
+
+## Specifying the Python version
+
+The Python version that is used defaults to whatever `python` program is in
+your `PATH`.   If PyCall can't find your Python (in which case `Pkg.add` will fail with an error message), or if you want to use a different version of Python on your system, you can change the Python version by setting the `PYTHON` environment variable and then re-running `Pkg.build("PyCall")`.  In Julia:
+
+    ENV["PYTHON"] = "... path of the python program you want ..."
+    Pkg.build("PyCall")
+
+Note also that you will need to re-run `Pkg.build("PyCall")` if your
+`python` program changes significantly (e.g. you switch to a new
+Python distro, or you switch from Python 2 to Python 3).
+
+The current Python version being used is stored in the `pyversion`
+global variable of the `PyCall` module.  You can also look at
+`PyCall.libpython` to find the name of the Python library or
+`PyCall.pyprogramname` for the `python` program name.
+
+(Technically, PyCall does not use the `python` program per se: it links
+directly to the `libpython` library.  But it finds the location of `libpython`
+by running `python` during `Pkg.build`.)
 
 ## Usage
 
@@ -98,8 +123,6 @@ Here are solutions to some common problems:
 
 * Sometimes calling a Python function fails because PyCall doesn't realize it is a callable object (since so many types of objects can be callable in Python).   The workaround is to use `pycall(foo, PyAny, args...)` instead of `foo(args...)`.   If you want to call `foo.bar(args...)` in Python, it is good to use `pycall(foo["bar"], PyAny, args...)`, where using `foo["bar"]` instead of `foo[:bar]` prevents any automatic conversion of the `bar` field.
 
-* If PyCall can't find the version of Python you want, try setting the `PYTHON` environment variable to the full pathname of the `python` executable.   Note that PyCall doesn't work properly with [Canopy/EPD Python](https://github.com/stevengj/PyCall.jl/issues/42) yet, and we recommend [Anaconda](https://store.continuum.io/cshop/anaconda/) instead.
-
 * By default, PyCall [doesn't include the current directory in the Python search path](https://github.com/stevengj/PyCall.jl/issues/48).   If you want to do that (in order to load a Python module from the current directory), just run `unshift!(PyVector(pyimport("sys")["path"]), "")`.
 
 ## Python object interfaces
@@ -260,49 +283,6 @@ and also by providing more type information to the Julia compiler.
   instead use `w.pymember(:member)` (for the `PyAny` conversion) or
   `w.pymember("member")` (for the raw `PyObject`).
 
-### Initialization
-
-By default, whenever you call any of the high-level PyCall routines
-above, the Python interpreter (corresponding to the `python`
-executable name) is initialized and remains in memory until Julia
-exits.
-
-However, you may want to modify this behavior to change the default
-Python version, to call low-level Python functions directly via
-`ccall`, or to free the memory consumed by Python.  This can be
-accomplished using:
-
-* PyCall uses the Python executable specified by the `PYTHON`
-  environment variable, or `"python"` if the environment variable
-  is not set, in order to determine what Python libraries to use.
-  You can set this environment variable as usual in your operating
-  system (e.g. in the Unix shell before running Julia), or from
-  within Julia via `ENV["PYTHON"] = "..."`.  Alternatively, you
-  can call `pyinitialize` (below) explicitly.
-
-* `pyinitialize(s::String)`: Initialize the Python interpreter using
-  the Python libraries corresponding to the `python` shared-library or
-  executable name given by the argument `s`.  Calling `pyinitialize()`
-  defaults to `pyinitialize(get(ENV,"PYTHON","python"))` as described
-  above, but you may need to change this in rare cases.  The
-  `pyinitialize` function *must* be called before you can call any
-  low-level Python functions (via `ccall`), but it is called
-  automatically as needed when you use the higher-level functions
-  above.  It is safe to call this function more than once; subsequent
-  calls will do nothing.
-
-* `pyfinalize()`: End the Python interpreter and free all associated
-  memory.  After this function is called, you *may no longer restart
-  Python* by calling `pyinitialize` again (an exception will be
-  thrown).  The reason is that some Python modules (e.g. numpy) crash
-  if their initialization routine is called more than once.
-  Subsequent calls to `pyfinalize` do nothing.  You must *not* try
-  to access any Python functions or data (that has not been *copied*
-  to native Julia types) after `pyfinalize` runs!
-
-* The Python version number is stored in the global variable
-  `pyversion::VersionNumber`.
-
 ### GUI Event Loops
 
 For Python packages that have a graphical user interface (GUI),
@@ -352,10 +332,10 @@ module](https://github.com/stevengj/PyPlot.jl) for Julia.
 ### Low-level Python API access
 
 If you want to call low-level functions in the Python C API, you can
-do so using `ccall`.  Just remember to call `pyinitialize()` first, and:
+do so using `ccall`. 
 
-* Use `pysym(func::Symbol)` to get a function pointer to pass to `ccall`
-  given a symbol `func` in the Python API.  e.g. you can call `int Py_IsInitialized()` by `ccall(pysym(:Py_IsInitialized), Int32, ())`.
+* Use `@pysym(func::Symbol)` to get a function pointer to pass to `ccall`
+  given a symbol `func` in the Python API.  e.g. you can call `int Py_IsInitialized()` by `ccall(@pysym(:Py_IsInitialized), Int32, ())`.
 
 * PyCall defines the typealias `PyPtr` for `PythonObject*` argument types,
   and `PythonObject` (see above) arguments are correctly converted to this

deps/build.jl

@@ -0,0 +1,186 @@
+# In this file, we figure out how to link to Python (surprisingly complicated)
+# and generate a deps/deps.jl file with the libpython name and other information
+# needed for static compilation of PyCall.
+
+# As a result, if you switch to a different version or path of Python, you
+# will probably need to re-run Pkg.build("PyCall").
+
+# remove deps.jl if it exists, in case build.jl fails
+isfile("deps.jl") && rm("deps.jl")
+
+using Compat
+
+#########################################################################
+
+# set PYTHONIOENCODING when running python executable, so that
+# we get UTF-8 encoded text as output (this is not the default on Windows).
+ENV["PYTHONIOENCODING"] = "UTF-8"
+
+pyconfigvar(python::AbstractString, var::AbstractString) = chomp(readall(`$python -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('$var'))"`))
+pyconfigvar(python, var, default) = let v = pyconfigvar(python, var)
+    v == "None" ? default : v
+end
+
+pysys(python::AbstractString, var::AbstractString) = chomp(readall(`$python -c "import sys; print(sys.$var)"`))
+
+#########################################################################
+
+const dlprefix = @windows? "" : "lib"
+
+# return libpython name, libpython pointer
+function find_libpython(python::AbstractString)
+    # it is ridiculous that it is this hard to find the name of libpython
+    v = pyconfigvar(python,"VERSION","")
+    libs = [ dlprefix*"python"*v*"."*Libdl.dlext, dlprefix*"python."*Libdl.dlext ]
+    lib = pyconfigvar(python, "LIBRARY")
+    lib != "None" && unshift!(libs, splitext(lib)[1]*"."*Libdl.dlext)
+    lib = pyconfigvar(python, "LDLIBRARY")
+    lib != "None" && unshift!(unshift!(libs, basename(lib)), lib)
+    libs = unique(libs)
+
+    libpaths = [pyconfigvar(python, "LIBDIR"),
+                (@windows ? dirname(pysys(python, "executable")) : joinpath(dirname(dirname(pysys(python, "executable"))), "lib"))]
+    @osx_only push!(libpaths, pyconfigvar(python, "PYTHONFRAMEWORKPREFIX"))
+
+    # `prefix` and `exec_prefix` are the path prefixes where python should look for python only and compiled libraries, respectively.
+    # These are also changed when run in a virtualenv.
+    exec_prefix = pyconfigvar(python, "exec_prefix")
+    # Since we only use `libpaths` to find the python dynamic library, we should only add `exec_prefix` to it.
+    push!(libpaths, exec_prefix)
+    if !haskey(ENV, "PYTHONHOME")
+        # PYTHONHOME tells python where to look for both pure python
+        # and binary modules.  When it is set, it replaces both
+        # `prefix` and `exec_prefix` and we thus need to set it to
+        # both in case they differ. This is also what the
+        # documentation recommends.  However, they are documented
+        # to always be the same on Windows, where it causes
+        # problems if we try to include both.
+        ENV["PYTHONHOME"] = @windows? exec_prefix : pyconfigvar(python, "prefix") * ":" * exec_prefix
+        # Unfortunately, setting PYTHONHOME screws up Canopy's Python distro
+        try
+            run(pipe(`$python -c "import site"`, stdout=DevNull, stderr=DevNull))
+        catch
+            pop!(ENV, "PYTHONHOME")
+        end
+    end
+    # TODO: look in python-config output? pyconfigvar("LDFLAGS")?
+    for lib in libs
+        for libpath in libpaths
+            libpath_lib = joinpath(libpath, lib)
+            if isfile(libpath_lib)
+                try
+                    return (Libdl.dlopen(libpath_lib,
+                                         Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL),
+                            libpath_lib)
+                end
+            end
+        end
+    end
+
+    # We do this *last* because the libpython in the system
+    # library path might be the wrong one if multiple python
+    # versions are installed (we prefer the one in LIBDIR):
+    for lib in libs
+        lib = splitext(lib)[1]
+        try
+            return (Libdl.dlopen(lib, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL),
+                    lib)
+        end
+    end
+    error("Couldn't find libpython; check your PYTHON environment variable")
+end
+
+#########################################################################
+
+hassym(lib, sym) = Libdl.dlsym_e(lib, sym) != C_NULL
+
+# call dlsym_e on a sequence of symbols and return the symbol that gives
+# the first non-null result
+function findsym(lib, syms...)
+    for sym in syms
+        if hassym(lib, sym)
+            return sym
+        end
+    end
+    error("no symbol found from: ", syms)
+end
+
+#########################################################################
+
+# need to be able to get the version before Python is initialized
+Py_GetVersion(libpy) = bytestring(ccall(Libdl.dlsym(libpy, :Py_GetVersion), Ptr{Uint8}, ()))
+
+const python = get(ENV, "PYTHON", "python")
+const (libpython, libpy_name) = find_libpython(python)
+const programname = pysys(python, "executable")
+
+# cache the Python version as a Julia VersionNumber
+const pyversion = convert(VersionNumber, split(Py_GetVersion(libpython))[1])
+
+if pyversion < v"2.7"
+    error("Python 2.7 or later is required for PyCall")
+end
+
+# PyUnicode_* may actually be a #define for another symbol, so
+# we cache the correct dlsym
+const PyUnicode_AsUTF8String =
+    findsym(libpython, :PyUnicode_AsUTF8String, :PyUnicodeUCS4_AsUTF8String, :PyUnicodeUCS2_AsUTF8String)
+const PyUnicode_DecodeUTF8 =
+    findsym(libpython, :PyUnicode_DecodeUTF8, :PyUnicodeUCS4_DecodeUTF8, :PyUnicodeUCS2_DecodeUTF8)
+
+# Python 2/3 compatibility: cache symbols for renamed functions
+if hassym(libpython, :PyString_FromString)
+    const PyString_FromString = :PyString_FromString
+    const PyString_AsString = :PyString_AsString
+    const PyString_Size = :PyString_Size
+    const PyString_Type = :PyString_Type
+else
+    const PyString_FromString = :PyBytes_FromString
+    const PyString_AsString = :PyBytes_AsString
+    const PyString_Size = :PyBytes_Size
+    const PyString_Type = :PyBytes_Type
+end
+if hassym(libpython, :PyInt_Type)
+    const PyInt_Type = :PyInt_Type
+    const PyInt_FromSize_t = :PyInt_FromSize_t
+    const PyInt_FromSsize_t = :PyInt_FromSsize_t
+    const PyInt_AsSsize_t = :PyInt_AsSsize_t
+else
+    const PyInt_Type = :PyLong_Type
+    const PyInt_FromSize_t = :PyLong_FromSize_t
+    const PyInt_FromSsize_t = :PyLong_FromSsize_t
+    const PyInt_AsSsize_t = :PyLong_AsSsize_t
+end
+
+# hashes changed from long to intptr_t in Python 3.2
+const Py_hash_t = pyversion < v"3.2" ? Clong:Int
+
+# whether to use unicode for strings by default, ala Python 3
+const pyunicode_literals = pyversion >= v"3.0"
+
+open("deps.jl", "w") do f
+    print(f, """
+          const python = "$(escape_string(python))"
+          const libpython = "$(escape_string(libpy_name))"
+          const pyprogramname = $(pyversion.major < 3 ? "bytestring" : "wstring")("$(escape_string(programname))")
+          const pyversion_build = $(repr(pyversion))
+
+          const PYTHONHOME = "$(escape_string(get(ENV, "PYTHONHOME", nothing)))"
+
+          const PyUnicode_AsUTF8String = :$PyUnicode_AsUTF8String
+          const PyUnicode_DecodeUTF8 = :$PyUnicode_DecodeUTF8
+
+          const PyString_FromString = :$PyString_FromString
+          const PyString_AsString = :$PyString_AsString
+          const PyString_Size = :$PyString_Size
+          const PyString_Type = :$PyString_Type
+          const PyInt_Type = :$PyInt_Type
+          const PyInt_FromSize_t = :$PyInt_FromSize_t
+          const PyInt_FromSsize_t = :$PyInt_FromSsize_t
+          const PyInt_AsSsize_t = :$PyInt_AsSsize_t
+
+          const Py_hash_t = $Py_hash_t
+          
+          const pyunicode_literals = $pyunicode_literals
+          """)
+end