Add interpreter constraints option and use constraints to search for compatible interpreters at exec time

.gitignore

@@ -10,3 +10,4 @@
 /.idea
 /.coverage*
 /htmlcov
+/.pyenv_test

.travis.yml

@@ -8,6 +8,10 @@ dist: precise
 
 # TRAVIS_PYTHON_VERSION
 
+cache:
+  directories:
+    - .pyenv_test
+
 matrix:
   include:
     - language: python

pex/bin/pex.py

@@ -23,6 +23,7 @@
 from pex.http import Context
 from pex.installer import EggInstaller
 from pex.interpreter import PythonInterpreter
+from pex.interpreter_constraints import check_requirements_are_well_formed, matched_interpreters
 from pex.iterator import Iterator
 from pex.package import EggPackage, SourcePackage
 from pex.pex import PEX
@@ -289,6 +290,17 @@ def configure_clp_pex_environment(parser):
            'can be passed multiple times to create a multi-interpreter compatible pex. '
            'Default: Use current interpreter.')
 
+  group.add_option(
+    '--interpreter-constraint',
+    dest='interpreter_constraint',
+    default=[],
+    type='str',
+    action='append',
+    help='A constraint that determines the interpreter compatibility for '
+         'this pex, using the Requirement-style format, e.g. "CPython>=3", or ">=2.7" '
+         'for requirements agnostic to interpreter class. This option can be passed multiple '
+         'times.')
+
   group.add_option(
       '--python-shebang',
       dest='python_shebang',
@@ -507,14 +519,6 @@ def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel)
     return interpreter
 
 
-def _lowest_version_interpreter(interpreters):
-  """Given a list of interpreters, return the one with the lowest version."""
-  lowest = interpreters[0]
-  for i in interpreters[1:]:
-    lowest = lowest if lowest < i else i
-  return lowest
-
-
 def build_pex(args, options, resolver_option_builder):
   with TRACER.timed('Resolving interpreters', V=2):
     interpreters = [
@@ -525,6 +529,12 @@ def build_pex(args, options, resolver_option_builder):
       for interpreter in options.python or [None]
     ]
 
+  if options.interpreter_constraint:
+    constraints = options.interpreter_constraint
+    # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted
+    check_requirements_are_well_formed(constraints)
+    interpreters = list(matched_interpreters(interpreters, constraints, meet_all_constraints=True))
+
   if not interpreters:
     die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
 
@@ -535,7 +545,8 @@ def build_pex(args, options, resolver_option_builder):
     # options.preamble_file is None
     preamble = None
 
-  interpreter = _lowest_version_interpreter(interpreters)
+  # TODO: make whether to take min or max version interpreter configurable
+  interpreter = min(interpreters)
   pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
 
   pex_info = pex_builder.info
@@ -544,6 +555,9 @@ def build_pex(args, options, resolver_option_builder):
   pex_info.always_write_cache = options.always_write_cache
   pex_info.ignore_errors = options.ignore_errors
   pex_info.inherit_path = options.inherit_path
+  if options.interpreter_constraint:
+    for ic in constraints:
+      pex_builder.add_interpreter_constraint(ic)
 
   resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]
 

pex/interpreter_constraints.py

@@ -0,0 +1,44 @@
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# A library of functions for filtering Python interpreters based on compatibility constraints
+
+from .interpreter import PythonIdentity
+
+
+def _matches(interpreter, filters, meet_all_constraints=False):
+  if meet_all_constraints:
+    return all(interpreter.identity.matches(filt) for filt in filters)
+  else:
+    return any(interpreter.identity.matches(filt) for filt in filters)
+
+
+def _matching(interpreters, filters, meet_all_constraints=False):
+  for interpreter in interpreters:
+    if _matches(interpreter, filters, meet_all_constraints):
+      yield interpreter
+
+
+def check_requirements_are_well_formed(constraints):
+  # Check that the compatibility requirements are well-formed.
+  for req in constraints:
+    try:
+      PythonIdentity.parse_requirement(req)
+    except ValueError as e:
+      from .common import die
+      die("Compatibility requirements are not formatted properly: %s" % str(e))
+
+
+def matched_interpreters(interpreters, filters, meet_all_constraints=False):
+  """Given some filters, yield any interpreter that matches at least one of them, or all of them
+     if meet_all_constraints is set to True.
+
+  :param interpreters: a list of PythonInterpreter objects for filtering
+  :param filters: A sequence of strings that constrain the interpreter compatibility for this
+    pex, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']``
+    for requirements agnostic to interpreter class.
+  :param meet_all_constraints: whether to match against all filters.
+    Defaults to matching interpreters that match at least one filter.
+  """
+  for match in _matching(interpreters, filters, meet_all_constraints):
+    yield match

pex/pex_bootstrapper.py

@@ -4,7 +4,12 @@
 import os
 import sys
 
-from .common import open_zip
+from .common import die, open_zip
+from .executor import Executor
+from .interpreter import PythonInterpreter
+from .interpreter_constraints import matched_interpreters
+from .tracer import TRACER
+from .variables import ENV
 
 __all__ = ('bootstrap_pex',)
 
@@ -56,28 +61,100 @@ def find_in_path(target_interpreter):
       return try_path
 
 
-def maybe_reexec_pex():
-  from .variables import ENV
-  if not ENV.PEX_PYTHON:
-    return
+def _get_python_interpreter(binary):
+  try:
+    return PythonInterpreter.from_binary(binary)
+  # from_binary attempts to execute the interpreter
+  except Executor.ExecutionError:
+    return None
+
+
+def _find_compatible_interpreters(pex_python_path, compatibility_constraints):
+  """Find all compatible interpreters on the system within the supplied constraints and use
+     PEX_PYTHON_PATH env variable if it is set. If not, fall back to interpreters on $PATH.
+  """
+  if pex_python_path:
+    interpreters = []
+    for binary in pex_python_path.split(os.pathsep):
+      pi = _get_python_interpreter(binary)
+      if pi:
+        interpreters.append(pi)
+    if not interpreters:
+      die('No interpreters from PEX_PYTHON_PATH can be found on the system. Exiting.')
+  else:
+    # All qualifying interpreters found in $PATH
+    interpreters = PythonInterpreter.all()
 
-  from .common import die
-  from .tracer import TRACER
+  compatible_interpreters = list(matched_interpreters(
+    interpreters, compatibility_constraints, meet_all_constraints=True))
+  return compatible_interpreters if compatible_interpreters else None
 
-  target_python = ENV.PEX_PYTHON
+
+def _handle_pex_python(target_python, compatibility_constraints):
+  """Re-exec using the PEX_PYTHON interpreter"""
   target = find_in_path(target_python)
   if not target:
     die('Failed to find interpreter specified by PEX_PYTHON: %s' % target)
+  elif compatibility_constraints:
+    pi = PythonInterpreter.from_binary(target)
+    if not all(pi.identity.matches(constraint) for constraint in compatibility_constraints):
+      die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified '
+          'interpreter constraints: %s' % (target, str(compatibility_constraints)))
   if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable):
     TRACER.log('Detected PEX_PYTHON, re-exec to %s' % target)
-    ENV.delete('PEX_PYTHON')
+    ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True
     os.execve(target, [target_python] + sys.argv, ENV.copy())
 
 
+def _handle_general_interpreter_selection(pex_python_path, compatibility_constraints):
+  """Handle selection in the case that PEX_PYTHON_PATH is set or interpreter compatibility
+     constraints are specified.
+  """
+  compatible_interpreters = _find_compatible_interpreters(
+    pex_python_path, compatibility_constraints)
+
+  target = None
+  if compatible_interpreters:
+    target = min(compatible_interpreters).binary
+  if not target:
+    die('Failed to find compatible interpreter for constraints: %s'
+        % str(compatibility_constraints))
+  if os.path.exists(target) and os.path.realpath(target) != os.path.realpath(sys.executable):
+    if pex_python_path:
+      TRACER.log('Detected PEX_PYTHON_PATH, re-exec to %s' % target)
+    else:
+      TRACER.log('Re-exec to interpreter %s that matches constraints: %s' % (target,
+        str(compatibility_constraints)))
+    ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC = True
+    os.execve(target, [target] + sys.argv, ENV.copy())
+
+
+def maybe_reexec_pex(compatibility_constraints=None):
+  if ENV.SHOULD_EXIT_BOOTSTRAP_REEXEC:
+    return
+  if ENV.PEX_PYTHON and ENV.PEX_PYTHON_PATH:
+    # both vars are defined, fall through to PEX_PYTHON_PATH resolution (give precedence to PPP)
+    pass
+  elif ENV.PEX_PYTHON:
+    # preserve PEX_PYTHON re-exec for backwards compatibility
+    _handle_pex_python(ENV.PEX_PYTHON, compatibility_constraints)
+
+  if not compatibility_constraints:
+    # if no compatibility constraints are specified, we want to match against
+    # the lowest-versioned interpreter in PEX_PYTHON_PATH if it is set
+    if not ENV.PEX_PYTHON_PATH:
+      # no PEX_PYTHON_PATH, PEX_PYTHON, or interpreter constraints, continue as normal
+      return
+
+  _handle_general_interpreter_selection(ENV.PEX_PYTHON_PATH, compatibility_constraints)
+
+
 def bootstrap_pex(entry_point):
   from .finders import register_finders
   register_finders()
-  maybe_reexec_pex()
+  pex_info = get_pex_info(entry_point)
+  with TRACER.timed('Bootstrapping pex with maybe_reexec_pex', V=2):
+    maybe_reexec_pex(pex_info.interpreter_constraints)
 
   from . import pex
   pex.PEX(entry_point).execute()

pex/pex_builder.py

@@ -166,6 +166,15 @@ def add_requirement(self, req):
     self._ensure_unfrozen('Adding a requirement')
     self._pex_info.add_requirement(req)
 
+  def add_interpreter_constraint(self, ic):
+    """Add an interpreter constraint to the PEX environment.
+
+    :param ic: A version constraint on the interpreter used to build and run this PEX environment.
+
+    """
+    self._ensure_unfrozen('Adding an interpreter constraint')
+    self._pex_info.add_interpreter_constraint(ic)
+
   def set_executable(self, filename, env_filename=None):
     """Set the executable for this environment.
 

pex/pex_info.py

@@ -123,6 +123,7 @@ def __init__(self, info=None):
                        '%s of type %s' % (info, type(info)))
     self._pex_info = info or {}
     self._distributions = self._pex_info.get('distributions', {})
+    self._interpreter_constraints = self._pex_info.get('interpreter_constraints', [])
     requirements = self._pex_info.get('requirements', [])
     if not isinstance(requirements, (list, tuple)):
       raise ValueError('Expected requirements to be a list, got %s' % type(requirements))
@@ -195,6 +196,20 @@ def inherit_path(self):
   def inherit_path(self, value):
     self._pex_info['inherit_path'] = bool(value)
 
+  @property
+  def interpreter_constraints(self):
+    """A list of constraints that determine the interpreter compatibility for this
+    pex, using the Requirement-style format, e.g. ``'CPython>=3', or just '>=2.7,<3'``
+    for requirements agnostic to interpreter class.
+
+    This property will be used at exec time when bootstrapping a pex to search PEX_PYTHON_PATH
+    for a list of compatible interpreters.
+    """
+    return self._interpreter_constraints
+
+  def add_interpreter_constraint(self, value):
+    self._interpreter_constraints.append(str(value))
+
   @property
   def ignore_errors(self):
     return self._pex_info.get('ignore_errors', False)
@@ -274,11 +289,13 @@ def update(self, other):
       raise TypeError('Cannot merge a %r with PexInfo' % type(other))
     self._pex_info.update(other._pex_info)
     self._distributions.update(other.distributions)
+    self._interpreter_constraints.extend(other.interpreter_constraints)
     self._requirements.update(other.requirements)
 
   def dump(self, **kwargs):
     pex_info_copy = self._pex_info.copy()
     pex_info_copy['requirements'] = list(self._requirements)
+    pex_info_copy['interpreter_constraints'] = self._interpreter_constraints
     pex_info_copy['distributions'] = self._distributions.copy()
     return json.dumps(pex_info_copy, **kwargs)
 

pex/testing.py

@@ -4,6 +4,7 @@
 import contextlib
 import os
 import random
+import subprocess
 import sys
 import tempfile
 from collections import namedtuple
@@ -277,3 +278,25 @@ def combine_pex_coverage(coverage_file_iter):
 
   combined.write()
   return combined.filename
+
+
+def bootstrap_python_installer():
+  install_location = os.getcwd() + '/.pyenv_test'
+  if not os.path.exists(install_location) or not os.path.exists(install_location + '/bin/pyenv'):
+    for _ in range(3):
+      try:
+        subprocess.call(['git', 'clone', 'https://github.com/pyenv/pyenv.git', install_location])
+      except StandardError:
+        continue
+      else:
+        break
+    else:
+      raise RuntimeError("Helper method could not clone pyenv from git")
+
+
+def ensure_python_interpreter(version):
+  bootstrap_python_installer()
+  install_location = os.getcwd() + '/.pyenv_test/versions/' + version
+  if not os.path.exists(install_location):
+    os.environ['PYENV_ROOT'] = os.getcwd() + '/.pyenv_test'
+    subprocess.call([os.getcwd() + '/.pyenv_test/bin/pyenv', 'install', version])

pex/variables.py

@@ -231,6 +231,18 @@ def PEX_PYTHON(self):
     """
     return self._get_string('PEX_PYTHON', default=None)
 
+  @property
+  def PEX_PYTHON_PATH(self):
+    """String
+
+    A colon-seperated string containing paths to binaires of blessed Python interpreters
+    for overriding the Python interpreter used to invoke this PEX. Must be absolute paths to the
+    interpreter.
+
+    Ex: "/path/to/python27:/path/to/python36"
+    """
+    return self._get_string('PEX_PYTHON_PATH', default=None)
+
   @property
   def PEX_ROOT(self):
     """Directory
@@ -300,6 +312,27 @@ def PEX_IGNORE_RCFILES(self):
     """
     return self._get_bool('PEX_IGNORE_RCFILES', default=False)
 
+  @property
+  def SHOULD_EXIT_BOOTSTRAP_REEXEC(self):
+    """Boolean
+
+    Whether to re-exec in maybe_reexec_pex function of pex_bootstrapper.py. Default: false.
+    This is neccesary because that function relies on checking against variables present in the
+    ENV to determine whether to re-exec or return to current execution. When we introduced
+    interpreter constraints to a pex, these constraints can influence interpreter selection without
+    the need for a PEX_PYTHON or PEX_PYTHON_PATH ENV variable set. Since re-exec previously checked
+    for PEX_PYTHON or PEX_PYTHON_PATH but not constraints, this would result in a loop with no
+    stopping criteria. Setting SHOULD_EXIT_BOOTSTRAP_REEXEC will let the runtime know to break out
+    of a second execution of maybe_reexec_pex if neither PEX_PYTHON or PEX_PYTHON_PATH are set but
+    interpreter constraints are specified.
+    """
+    value = self._environ.get('SHOULD_EXIT_REEXEC', 'False')
+    return False if value == 'False' else True
+
+  @SHOULD_EXIT_BOOTSTRAP_REEXEC.setter
+  def SHOULD_EXIT_BOOTSTRAP_REEXEC(self, value):
+    self._environ['SHOULD_EXIT_REEXEC'] = str(value)
+
 
 # Global singleton environment
 ENV = Variables()