Add interpreter constraints option and use constraints to search for compatible interpreters at exec time

.gitignore

@@ -10,3 +10,4 @@
 /.idea
 /.coverage*
 /htmlcov
+/.pyenv_test

.travis.yml

@@ -8,6 +8,16 @@ dist: precise
 
 # TRAVIS_PYTHON_VERSION
 
+env:
+  global:
+    # This is necessary to have consistent testing of methods that rely on falling back to PATH for
+    # selecting a python interpreter.
+    - PATH=/home/travis/build/pantsbuild/pex/.tox/py36/bin:/home/travis/build/pantsbuild/pex/.tox/py27/bin
+
+cache:
+  directories:
+    - .pyenv_test
+
 matrix:
   include:
     - language: python

pex/bin/pex.py

@@ -23,9 +23,11 @@
 from pex.http import Context
 from pex.installer import EggInstaller
 from pex.interpreter import PythonInterpreter
+from pex.interpreter_constraints import validate_constraints
 from pex.iterator import Iterator
 from pex.package import EggPackage, SourcePackage
 from pex.pex import PEX
+from pex.pex_bootstrapper import find_compatible_interpreters
 from pex.pex_builder import PEXBuilder
 from pex.platforms import Platform
 from pex.requirements import requirements_from_file
@@ -289,6 +291,25 @@ def configure_clp_pex_environment(parser):
            'can be passed multiple times to create a multi-interpreter compatible pex. '
            'Default: Use current interpreter.')
 
+  group.add_option(
+    '--interpreter-constraint',
+    dest='interpreter_constraint',
+    default=[],
+    type='str',
+    action='append',
+    help='A constraint that determines the interpreter compatibility for '
+         'this pex, using the Requirement-style format, e.g. "CPython>=3", or ">=2.7" '
+         'for requirements agnostic to interpreter class. This option can be passed multiple '
+         'times.')
+
+  group.add_option(
+    '--rcfile',
+    dest='rc_file',
+    default=None,
+    help='A path to a pexrc file. NOTE: this flag is for testing purposes only. It is to '
+         'be used in the case that a pexrc lives in the same directory as the created '
+         'output pex as is the case with the pex integration tests.')
+
   group.add_option(
       '--python-shebang',
       dest='python_shebang',
@@ -507,14 +528,6 @@ def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel)
     return interpreter
 
 
-def _lowest_version_interpreter(interpreters):
-  """Given a list of interpreters, return the one with the lowest version."""
-  lowest = interpreters[0]
-  for i in interpreters[1:]:
-    lowest = lowest if lowest < i else i
-  return lowest
-
-
 def build_pex(args, options, resolver_option_builder):
   with TRACER.timed('Resolving interpreters', V=2):
     interpreters = [
@@ -525,6 +538,18 @@ def build_pex(args, options, resolver_option_builder):
       for interpreter in options.python or [None]
     ]
 
+  if options.interpreter_constraint:
+    # NB: options.python and interpreter constraints cannot be used together, so this will not
+    # affect usages of the interpreter(s) specified by the "--python" command line flag.
+    constraints = options.interpreter_constraint
+    validate_constraints(constraints)
+    pexrc = None
+    if options.rc_file:
+      pexrc = options.rc_file
+    rc_variables = Variables.from_rc(rc=pexrc)
+    pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
+    interpreters = find_compatible_interpreters(pex_python_path, constraints)
+
   if not interpreters:
     die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
 
@@ -535,7 +560,8 @@ def build_pex(args, options, resolver_option_builder):
     # options.preamble_file is None
     preamble = None
 
-  interpreter = _lowest_version_interpreter(interpreters)
+  interpreter = min(interpreters)
+
   pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
 
   pex_info = pex_builder.info
@@ -544,6 +570,9 @@ def build_pex(args, options, resolver_option_builder):
   pex_info.always_write_cache = options.always_write_cache
   pex_info.ignore_errors = options.ignore_errors
   pex_info.inherit_path = options.inherit_path
+  if options.interpreter_constraint:
+    for ic in options.interpreter_constraint:
+      pex_builder.add_interpreter_constraint(ic)
 
   resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]
 
@@ -605,6 +634,9 @@ def main(args=None):
     args, cmdline = args, []
 
   options, reqs = parser.parse_args(args=args)
+  if options.python and options.interpreter_constraint:
+    die('The "--python" and "--interpreter-constraint" options cannot be used together.')
+
   if options.pex_root:
     ENV.set('PEX_ROOT', options.pex_root)
   else:

pex/interpreter_constraints.py

@@ -0,0 +1,37 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# A library of functions for filtering Python interpreters based on compatibility constraints
+
+from .common import die
+from .interpreter import PythonIdentity
+from .tracer import TRACER
+
+
+def validate_constraints(constraints):
+  # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted
+  # Check that the compatibility requirements are well-formed.
+  for req in constraints:
+    try:
+      PythonIdentity.parse_requirement(req)
+    except ValueError as e:
+      die("Compatibility requirements are not formatted properly: %s" % str(e))
+
+
+def matched_interpreters(interpreters, constraints, meet_all_constraints=False):
+  """Given some filters, yield any interpreter that matches at least one of them, or all of them
+     if meet_all_constraints is set to True.
+
+  :param interpreters: a list of PythonInterpreter objects for filtering
+  :param constraints: A sequence of strings that constrain the interpreter compatibility for this
+    pex, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']``
+    for requirements agnostic to interpreter class.
+  :param meet_all_constraints: whether to match against all filters.
+    Defaults to matching interpreters that match at least one filter.
+  """
+  check = all if meet_all_constraints else any
+  for interpreter in interpreters:
+    if check(interpreter.identity.matches(filt) for filt in constraints):
+      TRACER.log("Constraints on interpreters: %s, Matching Interpreter: %s"
+                 % (constraints, interpreter.binary), V=3)
+      yield interpreter

pex/pex_bootstrapper.py

@@ -4,7 +4,12 @@
 import os
 import sys
 
-from .common import open_zip
+from .common import die, open_zip
+from .executor import Executor
+from .interpreter import PythonInterpreter
+from .interpreter_constraints import matched_interpreters
+from .tracer import TRACER
+from .variables import ENV
 
 __all__ = ('bootstrap_pex',)
 
@@ -56,28 +61,107 @@ def find_in_path(target_interpreter):
       return try_path
 
 
-def maybe_reexec_pex():
-  from .variables import ENV
-  if not ENV.PEX_PYTHON:
-    return
+def find_compatible_interpreters(pex_python_path, compatibility_constraints):
+  """Find all compatible interpreters on the system within the supplied constraints and use
+     PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH.
+  """
+  if pex_python_path:
+    interpreters = []
+    for binary in pex_python_path.split(os.pathsep):
+      try:
+        interpreters.append(PythonInterpreter.from_binary(binary))
+      except Executor.ExecutionError:
+        TRACER.log("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary)
+    if not interpreters:
+      die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.')
+  else:
+    if not os.getenv('PATH', ''):
+      # no $PATH, use sys.executable
+      interpreters = [PythonInterpreter.get()]
+    else:
+      # get all qualifying interpreters found in $PATH
+      interpreters = PythonInterpreter.all()
+
+  return list(matched_interpreters(
+    interpreters, compatibility_constraints, meet_all_constraints=True))
 
-  from .common import die
-  from .tracer import TRACER
 
-  target_python = ENV.PEX_PYTHON
+def _select_pex_python_interpreter(target_python, compatibility_constraints):
   target = find_in_path(target_python)
+
   if not target:
     die('Failed to find interpreter specified by PEX_PYTHON: %s' % target)
+  if compatibility_constraints:
+    pi = PythonInterpreter.from_binary(target)
+    if not list(matched_interpreters([pi], compatibility_constraints, meet_all_constraints=True)):
+      die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified '
+          'interpreter constraints: %s' % (target, str(compatibility_constraints)))
+  if not os.path.exists(target):
+    die('Target interpreter specified by PEX_PYTHON %s does not exist. Exiting.' % target)
+  return target
+
+
+def _select_interpreter(pex_python_path, compatibility_constraints):
+  compatible_interpreters = find_compatible_interpreters(
+    pex_python_path, compatibility_constraints)
+
+  if not compatible_interpreters:
+    die('Failed to find compatible interpreter for constraints: %s'
+        % str(compatibility_constraints))
+  # TODO: https://github.com/pantsbuild/pex/issues/430
+  target = min(compatible_interpreters).binary
+
   if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable):
-    TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target)
+    return target
+
+
+def maybe_reexec_pex(compatibility_constraints):
+  """
+  Handle environment overrides for the Python interpreter to use when executing this pex.
+
+  This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO
+  metadata. If PEX_PYTHON is set in a pexrc, it attempts to obtain the binary location of the
+  interpreter specified by PEX_PYTHON. If PEX_PYTHON_PATH is set, it attempts to search the path for
+  a matching interpreter in accordance with the interpreter constraints. If both variables are
+  present in a pexrc, this function gives precedence to PEX_PYTHON_PATH and errors out if no
+  compatible interpreters can be found on said path. If neither variable is set, fall through to
+  plain pex execution using PATH searching or the currently executing interpreter.
+
+  :param compatibility_constraints: list of requirements-style strings that constrain the
+  Python interpreter to re-exec this pex with.
+
+  """
+  if ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC:
+    return
+
+  selected_interpreter = None
+  with TRACER.timed('Selecting runtime interpreter based on pexrc', V=3):
+    if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH:
+      # preserve PEX_PYTHON re-exec for backwards compatibility
+      # TODO: Kill this off completely in favor of PEX_PYTHON_PATH
+      # https://github.com/pantsbuild/pex/issues/431
+      selected_interpreter = _select_pex_python_interpreter(ENV.PEX_PYTHON,
+                                                            compatibility_constraints)
+    elif ENV.PEX_PYTHON_PATH:
+      selected_interpreter = _select_interpreter(ENV.PEX_PYTHON_PATH, compatibility_constraints)
+
+  if selected_interpreter:
     ENV.delete('PEX_PYTHON')
-    os.execve(target, [target_python] + sys.argv, ENV.copy())
+    ENV.delete('PEX_PYTHON_PATH')
+    ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True
+    cmdline = [selected_interpreter] + sys.argv[1:]
+    TRACER.log('Re-executing: cmdline="%s", sys.executable="%s", PEX_PYTHON="%s", '
+               'PEX_PYTHON_PATH="%s", COMPATIBILITY_CONSTRAINTS="%s"'
+               % (cmdline, sys.executable, ENV.PEX_PYTHON, ENV.PEX_PYTHON_PATH,
+                  compatibility_constraints))
+    os.execve(selected_interpreter, cmdline, ENV.copy())
 
 
 def bootstrap_pex(entry_point):
   from .finders import register_finders
   register_finders()
-  maybe_reexec_pex()
+  pex_info = get_pex_info(entry_point)
+  maybe_reexec_pex(pex_info.interpreter_constraints)
 
   from . import pex
   pex.PEX(entry_point).execute()

pex/pex_builder.py

@@ -76,14 +76,14 @@ def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, prea
       interpreter exit.
     """
     self._chroot = chroot or Chroot(path or safe_mkdtemp())
-    self._pex_info = pex_info or PexInfo.default()
     self._frozen = False
     self._interpreter = interpreter or PythonInterpreter.get()
     self._shebang = self._interpreter.identity.hashbang()
     self._logger = logging.getLogger(__name__)
     self._preamble = to_bytes(preamble or '')
     self._copy = copy
     self._distributions = set()
+    self._pex_info = pex_info or PexInfo.default(interpreter)
 
   def _ensure_unfrozen(self, name='Operation'):
     if self._frozen:
@@ -166,6 +166,15 @@ def add_requirement(self, req):
     self._ensure_unfrozen('Adding a requirement')
     self._pex_info.add_requirement(req)
 
+  def add_interpreter_constraint(self, ic):
+    """Add an interpreter constraint to the PEX environment.
+
+    :param ic: A version constraint on the interpreter used to build and run this PEX environment.
+
+    """
+    self._ensure_unfrozen('Adding an interpreter constraint')
+    self._pex_info.add_interpreter_constraint(ic)
+
   def set_executable(self, filename, env_filename=None):
     """Set the executable for this environment.
 

pex/pex_info.py

@@ -51,23 +51,23 @@ class PexInfo(object):
   INTERNAL_CACHE = '.deps'
 
   @classmethod
-  def make_build_properties(cls):
+  def make_build_properties(cls, interpreter=None):
     from .interpreter import PythonInterpreter
     from pkg_resources import get_platform
 
-    pi = PythonInterpreter.get()
+    pi = interpreter or PythonInterpreter.get()
     return {
       'class': pi.identity.interpreter,
       'version': pi.identity.version,
       'platform': get_platform(),
     }
 
   @classmethod
-  def default(cls):
+  def default(cls, interpreter=None):
     pex_info = {
       'requirements': [],
       'distributions': {},
-      'build_properties': cls.make_build_properties(),
+      'build_properties': cls.make_build_properties(interpreter),
     }
     return cls(info=pex_info)
 
@@ -123,6 +123,8 @@ def __init__(self, info=None):
                        '%s of type %s' % (info, type(info)))
     self._pex_info = info or {}
     self._distributions = self._pex_info.get('distributions', {})
+    # cast as set because pex info from json must store interpreter_constraints as a list
+    self._interpreter_constraints = set(self._pex_info.get('interpreter_constraints', set()))
     requirements = self._pex_info.get('requirements', [])
     if not isinstance(requirements, (list, tuple)):
       raise ValueError('Expected requirements to be a list, got %s' % type(requirements))
@@ -195,6 +197,20 @@ def inherit_path(self):
   def inherit_path(self, value):
     self._pex_info['inherit_path'] = bool(value)
 
+  @property
+  def interpreter_constraints(self):
+    """A list of constraints that determine the interpreter compatibility for this
+    pex, using the Requirement-style format, e.g. ``'CPython>=3', or just '>=2.7,<3'``
+    for requirements agnostic to interpreter class.
+
+    This property will be used at exec time when bootstrapping a pex to search PEX_PYTHON_PATH
+    for a list of compatible interpreters.
+    """
+    return list(self._interpreter_constraints)
+
+  def add_interpreter_constraint(self, value):
+    self._interpreter_constraints.add(str(value))
+
   @property
   def ignore_errors(self):
     return self._pex_info.get('ignore_errors', False)
@@ -274,11 +290,13 @@ def update(self, other):
       raise TypeError('Cannot merge a %r with PexInfo' % type(other))
     self._pex_info.update(other._pex_info)
     self._distributions.update(other.distributions)
+    self._interpreter_constraints.update(other.interpreter_constraints)
     self._requirements.update(other.requirements)
 
   def dump(self, **kwargs):
     pex_info_copy = self._pex_info.copy()
     pex_info_copy['requirements'] = list(self._requirements)
+    pex_info_copy['interpreter_constraints'] = list(self._interpreter_constraints)
     pex_info_copy['distributions'] = self._distributions.copy()
     return json.dumps(pex_info_copy, **kwargs)