From 46338fef0ed60d5d8154448cb8ccaf8c49726338 Mon Sep 17 00:00:00 2001 From: Brian Wickman Date: Tue, 14 Apr 2015 14:36:01 -0400 Subject: [PATCH] Add --python-shebang and PEX_PYTHON. Addresses #53. * Adds ``--python-shebang`` option to the pex tool in order to set the ``#!`` shebang to an exact path. * Adds support for ``PEX_PYTHON`` environment variable which will cause the pex file to reinvoke itself using the interpreter specified, e.g. ``PEX_PYTHON=python3.4`` or ``PEX_PYTHON=/exact/path/to/interpreter``. --- CHANGES.rst | 11 +++++++++++ pex/bin/pex.py | 11 +++++++++++ pex/environment.py | 13 ++++++++----- pex/pex_bootstrapper.py | 28 ++++++++++++++++++++++++++++ pex/pex_builder.py | 15 ++++++++++++++- pex/version.py | 2 +- tests/test_pex_binary.py | 3 +++ tests/test_pex_builder.py | 13 +++++++++++++ 8 files changed, 89 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index abff608fe..c210c278f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,17 @@ CHANGES ======= +---------- +1.0.0.dev3 +---------- + +* Adds ``--python-shebang`` option to the pex tool in order to set the ``#!`` shebang to an exact + path. + +* Adds support for ``PEX_PYTHON`` environment variable which will cause the pex file to reinvoke + itself using the interpreter specified, e.g. ``PEX_PYTHON=python3.4`` or + ``PEX_PYTHON=/exact/path/to/interpreter``. + ---------- 1.0.0.dev2 ---------- diff --git a/pex/bin/pex.py b/pex/bin/pex.py index df5769ed7..af41acb71 100644 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -228,6 +228,14 @@ def configure_clp_pex_environment(parser): 'path to an interpreter, or specify a binary accessible on $PATH. ' 'Default: Use current interpreter.') + group.add_option( + '--python-shebang', + dest='python_shebang', + default=None, + help='The exact shebang (#!...) line to add at the top of the PEX file minus the ' + '#!. This overrides the default behavior, which picks an environment python ' + 'interpreter compatible with the one used to build the PEX file.') + group.add_option( '--platform', dest='platform', @@ -446,6 +454,9 @@ def build_pex(args, options, resolver_option_builder): elif options.script: pex_builder.set_script(options.script) + if options.python_shebang: + pex_builder.set_shebang(options.python_shebang) + return pex_builder diff --git a/pex/environment.py b/pex/environment.py index dba6762fc..d20bd2210 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -17,7 +17,7 @@ find_distributions ) -from .common import open_zip, safe_mkdir, safe_rmtree +from .common import die, open_zip, safe_mkdir, safe_rmtree from .interpreter import PythonInterpreter from .package import distribution_compatible from .pex_builder import PEXBuilder @@ -152,10 +152,13 @@ def _activate(self): resolved = working_set.resolve(all_reqs, env=self) except DistributionNotFound as e: TRACER.log('Failed to resolve a requirement: %s' % e) - TRACER.log('Current working set:') - for dist in working_set: - TRACER.log(' - %s' % dist) - raise + TRACER.log('Distributions contained within this pex:') + if not self._pex_info.distributions: + TRACER.log(' None') + else: + for dist in self._pex_info.distributions: + TRACER.log(' - %s' % dist) + die('Failed to execute PEX file, missing compatible dependency for %s' % e) for dist in resolved: with TRACER.timed('Activating %s' % dist): diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index d47683e08..5fd47a887 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -3,6 +3,7 @@ import contextlib import os +import sys import zipfile __all__ = ('bootstrap_pex',) @@ -59,11 +60,38 @@ def memoized_build_zipmanifest(archive, memo={}): pkg_resources.build_zipmanifest = memoized_build_zipmanifest +def find_in_path(target_interpreter): + if os.path.exists(target_interpreter): + return target_interpreter + + for directory in os.getenv('PATH', '').split(os.pathsep): + try_path = os.path.join(directory, target_interpreter) + if os.path.exists(try_path): + return try_path + + +def maybe_reexec_pex(target_interpreter): + from .common import die + from .tracer import TRACER + target = find_in_path(target_interpreter) + if not target: + die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) + current = os.path.realpath(sys.executable) + if os.path.exists(target) and target != current: + TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target) + del os.environ['PEX_PYTHON'] + os.execve(target, [target_interpreter] + sys.argv, os.environ) + + def bootstrap_pex(entry_point): from .finders import register_finders monkeypatch_build_zipmanifest() register_finders() + python_env = os.getenv('PEX_PYTHON') + if python_env: + maybe_reexec_pex(python_env) + from . import pex pex.PEX(entry_point).execute() diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 89b5bde3d..5c33b66de 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -76,6 +76,7 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea self._pex_info = pex_info or PexInfo.default() self._frozen = False self._interpreter = interpreter or PythonInterpreter.get() + self._shebang = self._interpreter.identity.hashbang() self._logger = logging.getLogger(__name__) self._preamble = to_bytes(preamble or '') self._distributions = set() @@ -240,6 +241,18 @@ def set_entry_point(self, entry_point): self._ensure_unfrozen('Setting an entry point') self._pex_info.entry_point = entry_point + def set_shebang(self, shebang): + """Set the exact shebang line for the PEX file. + + For example, pex_builder.set_shebang('/home/wickman/Local/bin/python3.4'). This is + used to override the default behavior which is to have a #!/usr/bin/env line referencing an + interpreter compatible with the one used to build the PEX. + + :param shebang: The shebang line minus the #!. + :type shebang: str + """ + self._shebang = '#!%s' % shebang + def _add_dist_dir(self, path, dist_name): for root, _, files in os.walk(path): for f in files: @@ -402,7 +415,7 @@ def build(self, filename): safe_mkdir(os.path.dirname(filename)) with open(filename + '~', 'ab') as pexfile: assert os.path.getsize(pexfile.name) == 0 - pexfile.write(to_bytes('%s\n' % self._interpreter.identity.hashbang())) + pexfile.write(to_bytes('%s\n' % self._shebang)) self._chroot.zip(filename + '~', mode='a') if os.path.exists(filename): os.unlink(filename) diff --git a/pex/version.py b/pex/version.py index 8b56e9041..a6759023c 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,7 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = '1.0.0.dev2' +__version__ = '1.0.0.dev3' SETUPTOOLS_REQUIREMENT = 'setuptools>=2.2,<16' WHEEL_REQUIREMENT = 'wheel>=0.24.0,<0.25.0' diff --git a/tests/test_pex_binary.py b/tests/test_pex_binary.py index 1ad0a80a6..9d5be2956 100644 --- a/tests/test_pex_binary.py +++ b/tests/test_pex_binary.py @@ -1,3 +1,6 @@ +# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + from contextlib import contextmanager from optparse import OptionParser diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index d1c48d077..7eab1ef80 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -10,6 +10,7 @@ from pex.compatibility import nested from pex.pex import PEX +from pex.pex_builder import PEXBuilder from pex.testing import write_simple_pex as write_pex from pex.testing import make_bdist from pex.util import DistributionHelper @@ -52,3 +53,15 @@ def test_pex_builder(): assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success' + + +def test_pex_builder_shebang(): + pb = PEXBuilder() + pb.set_shebang('foobar') + + with temporary_dir() as td: + target = os.path.join(td, 'foo.pex') + pb.build(target) + expected_preamble = b'#!foobar\n' + with open(target, 'rb') as fp: + assert fp.read(len(expected_preamble)) == expected_preamble