diff --git a/.gitignore b/.gitignore index 1226a163e..b05ce0476 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /.idea /.coverage* /htmlcov +/.pyenv_test diff --git a/.travis.yml b/.travis.yml index 302983e0f..8a90ebbd2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,16 @@ dist: precise # TRAVIS_PYTHON_VERSION +env: + global: + # This is necessary to have consistent testing of methods that rely on falling back to PATH for + # selecting a python interpreter. + - PATH=/home/travis/build/pantsbuild/pex/.tox/py36/bin:/home/travis/build/pantsbuild/pex/.tox/py27/bin + +cache: + directories: + - .pyenv_test + matrix: include: - language: python diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 86a23cccb..af00c1c6e 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -23,9 +23,11 @@ from pex.http import Context from pex.installer import EggInstaller from pex.interpreter import PythonInterpreter +from pex.interpreter_constraints import validate_constraints from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage from pex.pex import PEX +from pex.pex_bootstrapper import find_compatible_interpreters from pex.pex_builder import PEXBuilder from pex.platforms import Platform from pex.requirements import requirements_from_file @@ -289,6 +291,24 @@ def configure_clp_pex_environment(parser): 'can be passed multiple times to create a multi-interpreter compatible pex. ' 'Default: Use current interpreter.') + group.add_option( + '--interpreter-constraint', + dest='interpreter_constraint', + default=[], + type='str', + action='append', + help='A constraint that determines the interpreter compatibility for ' + 'this pex, using the Requirement-style format, e.g. "CPython>=3", or ">=2.7" ' + 'for requirements agnostic to interpreter class. This option can be passed multiple ' + 'times.') + + group.add_option( + '--rcfile', + dest='rc_file', + default=None, + help='An additional path to a pexrc file to read during configuration parsing. ' + 'Used primarily for testing.') + group.add_option( '--python-shebang', dest='python_shebang', @@ -507,14 +527,6 @@ def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel) return interpreter -def _lowest_version_interpreter(interpreters): - """Given a list of interpreters, return the one with the lowest version.""" - lowest = interpreters[0] - for i in interpreters[1:]: - lowest = lowest if lowest < i else i - return lowest - - def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ @@ -525,6 +537,15 @@ def build_pex(args, options, resolver_option_builder): for interpreter in options.python or [None] ] + if options.interpreter_constraint: + # NB: options.python and interpreter constraints cannot be used together, so this will not + # affect usages of the interpreter(s) specified by the "--python" command line flag. + constraints = options.interpreter_constraint + validate_constraints(constraints) + rc_variables = Variables.from_rc(rc=options.rc_file) + pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') + interpreters = find_compatible_interpreters(pex_python_path, constraints) + if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) @@ -535,7 +556,8 @@ def build_pex(args, options, resolver_option_builder): # options.preamble_file is None preamble = None - interpreter = _lowest_version_interpreter(interpreters) + interpreter = min(interpreters) + pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info @@ -544,6 +566,9 @@ def build_pex(args, options, resolver_option_builder): pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path + if options.interpreter_constraint: + for ic in options.interpreter_constraint: + pex_builder.add_interpreter_constraint(ic) resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] @@ -605,6 +630,9 @@ def main(args=None): args, cmdline = args, [] options, reqs = parser.parse_args(args=args) + if options.python and options.interpreter_constraint: + die('The "--python" and "--interpreter-constraint" options cannot be used together.') + if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: diff --git a/pex/interpreter_constraints.py b/pex/interpreter_constraints.py new file mode 100644 index 000000000..dfd3dd2ef --- /dev/null +++ b/pex/interpreter_constraints.py @@ -0,0 +1,39 @@ +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# A library of functions for filtering Python interpreters based on compatibility constraints + +from .common import die +from .interpreter import PythonIdentity +from .tracer import TRACER + + +def validate_constraints(constraints): + # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted: + # https://github.com/pantsbuild/pex/issues/432 + for req in constraints: + # Check that the compatibility requirements are well-formed. + try: + PythonIdentity.parse_requirement(req) + except ValueError as e: + die("Compatibility requirements are not formatted properly: %s" % str(e)) + + +def matched_interpreters(interpreters, constraints, meet_all_constraints=False): + """Given some filters, yield any interpreter that matches at least one of them, or all of them + if meet_all_constraints is set to True. + + :param interpreters: a list of PythonInterpreter objects for filtering + :param constraints: A sequence of strings that constrain the interpreter compatibility for this + pex, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` + for requirements agnostic to interpreter class. + :param meet_all_constraints: whether to match against all filters. + Defaults to matching interpreters that match at least one filter. + :return interpreter: returns a generator that yields compatible interpreters + """ + check = all if meet_all_constraints else any + for interpreter in interpreters: + if check(interpreter.identity.matches(filt) for filt in constraints): + TRACER.log("Constraints on interpreters: %s, Matching Interpreter: %s" + % (constraints, interpreter.binary), V=3) + yield interpreter diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index c3510a84b..3c6aa3c45 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -1,10 +1,15 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). - +from __future__ import print_function import os import sys -from .common import open_zip +from .common import die, open_zip +from .executor import Executor +from .interpreter import PythonInterpreter +from .interpreter_constraints import matched_interpreters +from .tracer import TRACER +from .variables import ENV __all__ = ('bootstrap_pex',) @@ -56,28 +61,108 @@ def find_in_path(target_interpreter): return try_path -def maybe_reexec_pex(): - from .variables import ENV - if not ENV.PEX_PYTHON: - return +def find_compatible_interpreters(pex_python_path, compatibility_constraints): + """Find all compatible interpreters on the system within the supplied constraints and use + PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH. + """ + if pex_python_path: + interpreters = [] + for binary in pex_python_path.split(os.pathsep): + try: + interpreters.append(PythonInterpreter.from_binary(binary)) + except Executor.ExecutionError: + print("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary, + file=sys.stderr) + if not interpreters: + die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') + else: + if not os.getenv('PATH', ''): + # no $PATH, use sys.executable + interpreters = [PythonInterpreter.get()] + else: + # get all qualifying interpreters found in $PATH + interpreters = PythonInterpreter.all() + + return list(matched_interpreters( + interpreters, compatibility_constraints, meet_all_constraints=True)) - from .common import die - from .tracer import TRACER - target_python = ENV.PEX_PYTHON +def _select_pex_python_interpreter(target_python, compatibility_constraints): target = find_in_path(target_python) + if not target: die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) + if compatibility_constraints: + pi = PythonInterpreter.from_binary(target) + if not list(matched_interpreters([pi], compatibility_constraints, meet_all_constraints=True)): + die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified ' + 'interpreter constraints: %s' % (target, str(compatibility_constraints))) + if not os.path.exists(target): + die('Target interpreter specified by PEX_PYTHON %s does not exist. Exiting.' % target) + return target + + +def _select_interpreter(pex_python_path, compatibility_constraints): + compatible_interpreters = find_compatible_interpreters( + pex_python_path, compatibility_constraints) + + if not compatible_interpreters: + die('Failed to find compatible interpreter for constraints: %s' + % str(compatibility_constraints)) + # TODO: https://github.com/pantsbuild/pex/issues/430 + target = min(compatible_interpreters).binary + if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable): - TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target) + return target + + +def maybe_reexec_pex(compatibility_constraints): + """ + Handle environment overrides for the Python interpreter to use when executing this pex. + + This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO + metadata. If PEX_PYTHON is set in a pexrc, it attempts to obtain the binary location of the + interpreter specified by PEX_PYTHON. If PEX_PYTHON_PATH is set, it attempts to search the path for + a matching interpreter in accordance with the interpreter constraints. If both variables are + present in a pexrc, this function gives precedence to PEX_PYTHON_PATH and errors out if no + compatible interpreters can be found on said path. If neither variable is set, fall through to + plain pex execution using PATH searching or the currently executing interpreter. + + :param compatibility_constraints: list of requirements-style strings that constrain the + Python interpreter to re-exec this pex with. + + """ + if ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC: + return + + selected_interpreter = None + with TRACER.timed('Selecting runtime interpreter based on pexrc', V=3): + if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH: + # preserve PEX_PYTHON re-exec for backwards compatibility + # TODO: Kill this off completely in favor of PEX_PYTHON_PATH + # https://github.com/pantsbuild/pex/issues/431 + selected_interpreter = _select_pex_python_interpreter(ENV.PEX_PYTHON, + compatibility_constraints) + elif ENV.PEX_PYTHON_PATH: + selected_interpreter = _select_interpreter(ENV.PEX_PYTHON_PATH, compatibility_constraints) + + if selected_interpreter: ENV.delete('PEX_PYTHON') - os.execve(target, [target_python] + sys.argv, ENV.copy()) + ENV.delete('PEX_PYTHON_PATH') + ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True + cmdline = [selected_interpreter] + sys.argv[1:] + TRACER.log('Re-executing: cmdline="%s", sys.executable="%s", PEX_PYTHON="%s", ' + 'PEX_PYTHON_PATH="%s", COMPATIBILITY_CONSTRAINTS="%s"' + % (cmdline, sys.executable, ENV.PEX_PYTHON, ENV.PEX_PYTHON_PATH, + compatibility_constraints)) + os.execve(selected_interpreter, cmdline, ENV.copy()) def bootstrap_pex(entry_point): from .finders import register_finders register_finders() - maybe_reexec_pex() + pex_info = get_pex_info(entry_point) + maybe_reexec_pex(pex_info.interpreter_constraints) from . import pex pex.PEX(entry_point).execute() diff --git a/pex/pex_builder.py b/pex/pex_builder.py index 36b4056d5..dd9c9a028 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -76,7 +76,6 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea interpreter exit. """ self._chroot = chroot or Chroot(path or safe_mkdtemp()) - self._pex_info = pex_info or PexInfo.default() self._frozen = False self._interpreter = interpreter or PythonInterpreter.get() self._shebang = self._interpreter.identity.hashbang() @@ -84,6 +83,7 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea self._preamble = to_bytes(preamble or '') self._copy = copy self._distributions = set() + self._pex_info = pex_info or PexInfo.default(interpreter) def _ensure_unfrozen(self, name='Operation'): if self._frozen: @@ -166,6 +166,15 @@ def add_requirement(self, req): self._ensure_unfrozen('Adding a requirement') self._pex_info.add_requirement(req) + def add_interpreter_constraint(self, ic): + """Add an interpreter constraint to the PEX environment. + + :param ic: A version constraint on the interpreter used to build and run this PEX environment. + + """ + self._ensure_unfrozen('Adding an interpreter constraint') + self._pex_info.add_interpreter_constraint(ic) + def set_executable(self, filename, env_filename=None): """Set the executable for this environment. diff --git a/pex/pex_info.py b/pex/pex_info.py index e6a053116..eae310a9a 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -51,11 +51,11 @@ class PexInfo(object): INTERNAL_CACHE = '.deps' @classmethod - def make_build_properties(cls): + def make_build_properties(cls, interpreter=None): from .interpreter import PythonInterpreter from pkg_resources import get_platform - pi = PythonInterpreter.get() + pi = interpreter or PythonInterpreter.get() return { 'class': pi.identity.interpreter, 'version': pi.identity.version, @@ -63,11 +63,11 @@ def make_build_properties(cls): } @classmethod - def default(cls): + def default(cls, interpreter=None): pex_info = { 'requirements': [], 'distributions': {}, - 'build_properties': cls.make_build_properties(), + 'build_properties': cls.make_build_properties(interpreter), } return cls(info=pex_info) @@ -123,6 +123,8 @@ def __init__(self, info=None): '%s of type %s' % (info, type(info))) self._pex_info = info or {} self._distributions = self._pex_info.get('distributions', {}) + # cast as set because pex info from json must store interpreter_constraints as a list + self._interpreter_constraints = set(self._pex_info.get('interpreter_constraints', set())) requirements = self._pex_info.get('requirements', []) if not isinstance(requirements, (list, tuple)): raise ValueError('Expected requirements to be a list, got %s' % type(requirements)) @@ -195,6 +197,20 @@ def inherit_path(self): def inherit_path(self, value): self._pex_info['inherit_path'] = bool(value) + @property + def interpreter_constraints(self): + """A list of constraints that determine the interpreter compatibility for this + pex, using the Requirement-style format, e.g. ``'CPython>=3', or just '>=2.7,<3'`` + for requirements agnostic to interpreter class. + + This property will be used at exec time when bootstrapping a pex to search PEX_PYTHON_PATH + for a list of compatible interpreters. + """ + return list(self._interpreter_constraints) + + def add_interpreter_constraint(self, value): + self._interpreter_constraints.add(str(value)) + @property def ignore_errors(self): return self._pex_info.get('ignore_errors', False) @@ -274,11 +290,13 @@ def update(self, other): raise TypeError('Cannot merge a %r with PexInfo' % type(other)) self._pex_info.update(other._pex_info) self._distributions.update(other.distributions) + self._interpreter_constraints.update(other.interpreter_constraints) self._requirements.update(other.requirements) def dump(self, **kwargs): pex_info_copy = self._pex_info.copy() pex_info_copy['requirements'] = list(self._requirements) + pex_info_copy['interpreter_constraints'] = list(self._interpreter_constraints) pex_info_copy['distributions'] = self._distributions.copy() return json.dumps(pex_info_copy, **kwargs) diff --git a/pex/testing.py b/pex/testing.py index 215071eb4..964a8a3ae 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -4,6 +4,7 @@ import contextlib import os import random +import subprocess import sys import tempfile from collections import namedtuple @@ -277,3 +278,27 @@ def combine_pex_coverage(coverage_file_iter): combined.write() return combined.filename + + +def bootstrap_python_installer(): + install_location = os.path.join(os.getcwd(), '.pyenv_test') + if not os.path.exists(install_location) or not os.path.exists( + os.path.join(os.getcwd(), '.pyenv_test')): + for _ in range(3): + try: + subprocess.call(['git', 'clone', 'https://github.com/pyenv/pyenv.git', install_location]) + except StandardError: + continue + else: + break + else: + raise RuntimeError("Helper method could not clone pyenv from git") + + +def ensure_python_interpreter(version): + bootstrap_python_installer() + install_location = os.path.join(os.getcwd(), '.pyenv_test/versions', version) + if not os.path.exists(install_location): + os.environ['PYENV_ROOT'] = os.path.join(os.getcwd(), '.pyenv_test') + subprocess.call([os.path.join(os.getcwd(), '.pyenv_test/bin/pyenv'), 'install', version]) + return os.path.join(install_location, 'bin', 'python' + version[0:3]) diff --git a/pex/variables.py b/pex/variables.py index 38208754d..d1d3d5867 100644 --- a/pex/variables.py +++ b/pex/variables.py @@ -33,11 +33,40 @@ def iter_help(cls): variable_type, variable_text = cls.process_pydoc(getattr(value, '__doc__')) yield variable_name, variable_type, variable_text - def __init__(self, environ=None, rc='~/.pexrc', use_defaults=True): + @classmethod + def from_rc(cls, rc=None): + """Read pex runtime configuration variables from a pexrc file. + + :param rc: an absolute path to a pexrc file. + :return: A dict of key value pairs found in processed pexrc files. + :rtype: dict + """ + ret_vars = {} + rc_locations = ['/etc/pexrc', + '~/.pexrc', + os.path.join(os.path.dirname(sys.argv[0]), '.pexrc')] + if rc: + rc_locations.append(rc) + for filename in rc_locations: + try: + with open(os.path.expanduser(filename)) as fh: + rc_items = map(cls._get_kv, fh) + ret_vars.update(dict(filter(None, rc_items))) + except IOError: + continue + return ret_vars + + @classmethod + def _get_kv(cls, variable): + kv = variable.strip().split('=') + if len(list(filter(None, kv))) == 2: + return kv + + def __init__(self, environ=None, rc=None, use_defaults=True): self._use_defaults = use_defaults self._environ = environ.copy() if environ else os.environ if not self.PEX_IGNORE_RCFILES: - rc_values = self._from_rc(rc).copy() + rc_values = self.from_rc(rc).copy() rc_values.update(self._environ) self._environ = rc_values @@ -50,22 +79,6 @@ def delete(self, variable): def set(self, variable, value): self._environ[variable] = str(value) - def _from_rc(self, rc): - ret_vars = {} - for filename in ['/etc/pexrc', rc, os.path.join(os.path.dirname(sys.argv[0]), '.pexrc')]: - try: - with open(os.path.expanduser(filename)) as fh: - rc_items = map(self._get_kv, fh) - ret_vars.update(dict(filter(None, rc_items))) - except IOError: - continue - return ret_vars - - def _get_kv(self, variable): - kv = variable.strip().split('=') - if len(list(filter(None, kv))) == 2: - return kv - def _defaulted(self, default): return default if self._use_defaults else None @@ -231,6 +244,18 @@ def PEX_PYTHON(self): """ return self._get_string('PEX_PYTHON', default=None) + @property + def PEX_PYTHON_PATH(self): + """String + + A colon-separated string containing paths of blessed Python interpreters + for overriding the Python interpreter used to invoke this PEX. Must be absolute paths to the + interpreter. + + Ex: "/path/to/python27:/path/to/python36" + """ + return self._get_string('PEX_PYTHON_PATH', default=None) + @property def PEX_ROOT(self): """Directory @@ -300,6 +325,26 @@ def PEX_IGNORE_RCFILES(self): """ return self._get_bool('PEX_IGNORE_RCFILES', default=False) + @property + def SHOULD_EXIT_BOOTSTRAP_REEXEC(self): + """Boolean + + Whether to re-exec in maybe_reexec_pex function of pex_bootstrapper.py. Default: false. + This is necessary because that function relies on checking against variables present in the + ENV to determine whether to re-exec or return to current execution. When we introduced + interpreter constraints to a pex, these constraints can influence interpreter selection without + the need for a PEX_PYTHON or PEX_PYTHON_PATH ENV variable set. Since re-exec previously checked + for PEX_PYTHON or PEX_PYTHON_PATH but not constraints, this would result in a loop with no + stopping criteria. Setting SHOULD_EXIT_BOOTSTRAP_REEXEC will let the runtime know to break out + of a second execution of maybe_reexec_pex if neither PEX_PYTHON or PEX_PYTHON_PATH are set but + interpreter constraints are specified. + """ + return bool(self._environ.get('SHOULD_EXIT_BOOTSTRAP_REEXEC', '')) + + @SHOULD_EXIT_BOOTSTRAP_REEXEC.setter + def SHOULD_EXIT_BOOTSTRAP_REEXEC(self, value): + self._environ['SHOULD_EXIT_BOOTSTRAP_REEXEC'] = str(value) + # Global singleton environment ENV = Variables() diff --git a/tests/test_integration.py b/tests/test_integration.py index 58ece3a2c..c5ea290fd 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,7 +10,9 @@ from pex.compatibility import WINDOWS from pex.installer import EggInstaller +from pex.pex_bootstrapper import get_pex_info from pex.testing import ( + ensure_python_interpreter, get_dep_dist_names_from_pex, run_pex_command, run_simple_pex, @@ -326,3 +328,222 @@ def test_pex_path_in_pex_info_and_env(): stdout, rc = run_simple_pex(pex_out_path, [test_file_path], env=env) assert rc == 0 assert stdout == b'Success!\n' + + +def test_interpreter_constraints_to_pex_info(): + with temporary_dir() as output_dir: + # target python 2 + pex_out_path = os.path.join(output_dir, 'pex1.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>=2.7', + '--interpreter-constraint=<3', + '-o', pex_out_path]) + res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert set(['>=2.7', '<3']) == set(pex_info.interpreter_constraints) + + # target python 3 + pex_out_path = os.path.join(output_dir, 'pex1.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>3', + '-o', pex_out_path]) + res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert ['>3'] == pex_info.interpreter_constraints + + +def test_interpreter_resolution_with_constraint_option(): + with temporary_dir() as output_dir: + pex_out_path = os.path.join(output_dir, 'pex1.pex') + res = run_pex_command(['--disable-cache', + '--interpreter-constraint=>=2.7', + '--interpreter-constraint=<3', + '-o', pex_out_path]) + res.assert_success() + pex_info = get_pex_info(pex_out_path) + assert set(['>=2.7', '<3']) == set(pex_info.interpreter_constraints) + assert pex_info.build_properties['version'][0] < 3 + + +def test_interpreter_resolution_with_pex_python_path(): + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set pex python path + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) + pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) + + # constraints to build pex cleanly; PPP + pex_bootstrapper.py + # will use these constraints to override sys.executable on pex re-exec + interpreter_constraint1 = '>3' if sys.version_info[0] == 3 else '<3' + interpreter_constraint2 = '<3.8' if sys.version_info[0] == 3 else '>=2.7' + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, + '--interpreter-constraint=%s' % interpreter_constraint1, + '--interpreter-constraint=%s' % interpreter_constraint2, + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + + assert rc == 0 + if sys.version_info[0] == 3: + assert str(pex_python_path.split(':')[1]).encode() in stdout + else: + assert str(pex_python_path.split(':')[0]).encode() in stdout + + +@pytest.mark.skipif(NOT_CPYTHON_36) +def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set both PPP and PP + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) + pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path) + pex_python = '/path/to/some/python' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python_path.split(':')[1].encode() + assert correct_interpreter_path in stdout + + +def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): + with temporary_dir() as td: + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + assert str(sys.executable).encode() in stdout + + +def test_pex_exec_with_pex_python_path_only(): + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set pex python path + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) + pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path) + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, + '-o', pex_out_path]) + res.assert_success() + + # test that pex bootstrapper selects lowest version interpreter + # in pex python path (python2.7) + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + assert str(pex_python_path.split(':')[0]).encode() in stdout + + +def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + # set both PPP and PP + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('3.6.3') + ]) + pexrc.write("PEX_PYTHON_PATH=%s\n" % pex_python_path) + pex_python = '/path/to/some/python' + pexrc.write("PEX_PYTHON=%s" % pex_python) + + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, + '-o', pex_out_path]) + res.assert_success() + + # test that pex bootstrapper selects lowest version interpreter + # in pex python path (python2.7) + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + assert str(pex_python_path.split(':')[0]).encode() in stdout + + +def test_pex_python(): + with temporary_dir() as td: + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + pex_python = ensure_python_interpreter('3.6.3') + pexrc.write("PEX_PYTHON=%s" % pex_python) + + # test PEX_PYTHON with valid constraints + pex_out_path = os.path.join(td, 'pex.pex') + res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python.encode() + assert correct_interpreter_path in stdout + + # test PEX_PYTHON with incompatible constraints + pexrc_path = os.path.join(td, '.pexrc') + with open(pexrc_path, 'w') as pexrc: + pex_python = ensure_python_interpreter('2.7.10') + pexrc.write("PEX_PYTHON=%s" % pex_python) + + pex_out_path = os.path.join(td, 'pex2.pex') + res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, + '--interpreter-constraint=>3', + '--interpreter-constraint=<3.8', + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 1 + fail_str = 'not compatible with specified interpreter constraints'.encode() + assert fail_str in stdout + + # test PEX_PYTHON with no constraints + pex_out_path = os.path.join(td, 'pex3.pex') + res = run_pex_command(['--disable-cache', + '--rcfile=%s' % pexrc_path, + '-o', pex_out_path]) + res.assert_success() + + stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' + stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) + assert rc == 0 + correct_interpreter_path = pex_python.encode() + assert correct_interpreter_path in stdout diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index 27b38f7f9..9592e8fe2 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -6,8 +6,9 @@ from twitter.common.contextutil import temporary_dir from pex.common import open_zip -from pex.pex_bootstrapper import get_pex_info -from pex.testing import write_simple_pex +from pex.interpreter import PythonInterpreter +from pex.pex_bootstrapper import find_compatible_interpreters, get_pex_info +from pex.testing import ensure_python_interpreter, write_simple_pex def test_get_pex_info(): @@ -28,3 +29,42 @@ def test_get_pex_info(): # same when encoded assert pex_info.dump() == pex_info_2.dump() + + +def test_find_compatible_interpreters(): + pex_python_path = ':'.join([ + ensure_python_interpreter('2.7.9'), + ensure_python_interpreter('2.7.10'), + ensure_python_interpreter('2.7.11'), + ensure_python_interpreter('3.4.2'), + ensure_python_interpreter('3.5.4'), + ensure_python_interpreter('3.6.2'), + ensure_python_interpreter('3.6.3') + ]) + + interpreters = find_compatible_interpreters(pex_python_path, ['>3']) + assert interpreters[0].binary == pex_python_path.split(':')[3] # 3.4.2 + + interpreters = find_compatible_interpreters(pex_python_path, ['<3']) + assert interpreters[0].binary == pex_python_path.split(':')[0] # 2.7.9 + + interpreters = find_compatible_interpreters(pex_python_path, ['>3.5.4']) + assert interpreters[0].binary == pex_python_path.split(':')[5] # 3.6.2 + + interpreters = find_compatible_interpreters(pex_python_path, ['>3.4.2, <3.6']) + assert interpreters[0].binary == pex_python_path.split(':')[4] # 3.5.4 + + interpreters = find_compatible_interpreters(pex_python_path, ['>3.6.2']) + assert interpreters[0].binary == pex_python_path.split(':')[6] # 3.6.3 + + interpreters = find_compatible_interpreters(pex_python_path, ['<2']) + assert not interpreters + + interpreters = find_compatible_interpreters(pex_python_path, ['>4']) + assert not interpreters + + interpreters = find_compatible_interpreters(pex_python_path, ['<2.7.11, >2.7.9']) + assert interpreters[0].binary == pex_python_path.split(':')[1] # 2.7.10 + + interpreters = find_compatible_interpreters('', ['<3']) + assert interpreters[0] in PythonInterpreter.all() # All interpreters on PATH diff --git a/tests/test_variables.py b/tests/test_variables.py index 3a33fbede..9cc48fa27 100644 --- a/tests/test_variables.py +++ b/tests/test_variables.py @@ -95,7 +95,7 @@ def test_pexrc_precedence(): with named_temporary_file(mode='w') as pexrc: pexrc.write('HELLO=FORTYTWO') pexrc.flush() - v = Variables(environ={'HELLO': 42}, rc=pexrc.name) + v = Variables(rc=pexrc.name, environ={'HELLO': 42}) assert v._get_int('HELLO') == 42 @@ -103,7 +103,7 @@ def test_rc_ignore(): with named_temporary_file(mode='w') as pexrc: pexrc.write('HELLO=FORTYTWO') pexrc.flush() - v = Variables(environ={'PEX_IGNORE_RCFILES': 'True'}, rc=pexrc.name) + v = Variables(rc=pexrc.name, environ={'PEX_IGNORE_RCFILES': 'True'}) assert 'HELLO' not in v._environ