Setup interpreter extras in InstallerBase.

pex/bin/pex.py

@@ -13,7 +13,6 @@
 from optparse import OptionGroup, OptionParser, OptionValueError
 from textwrap import TextWrapper
 
-from pex import vendor
 from pex.common import die, safe_delete, safe_mkdtemp
 from pex.fetcher import Fetcher, PyPIFetcher
 from pex.interpreter import PythonInterpreter
@@ -524,11 +523,7 @@ def to_python_interpreter(full_path_or_basename):
     pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
     interpreters = find_compatible_interpreters(pex_python_path, constraints)
 
-  setup_interpreters = [vendor.setup_interpreter(interpreter=interp,
-                                                 include_wheel=options.use_wheel)
-                        for interp in interpreters]
-
-  if not setup_interpreters:
+  if not interpreters:
     die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)
 
   try:
@@ -538,7 +533,7 @@ def to_python_interpreter(full_path_or_basename):
     # options.preamble_file is None
     preamble = None
 
-  interpreter = min(setup_interpreters)
+  interpreter = min(interpreters)
 
   pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
 
@@ -587,7 +582,7 @@ def walk_and_do(fn, src_dir):
   with TRACER.timed('Resolving distributions'):
     try:
       resolveds = resolve_multi(resolvables,
-                                interpreters=setup_interpreters,
+                                interpreters=interpreters,
                                 platforms=options.platforms,
                                 cache=options.cache_dir,
                                 cache_ttl=options.cache_ttl,

pex/installer.py

@@ -37,7 +37,10 @@ def __init__(self, source_dir, interpreter=None, install_dir=None):
     self._source_dir = source_dir
     self._install_tmp = install_dir or safe_mkdtemp()
     self._installed = None
-    self._interpreter = interpreter or PythonInterpreter.get()
+
+    from pex import vendor
+    self._interpreter = vendor.setup_interpreter(distributions=self.mixins,
+                                                 interpreter=interpreter or PythonInterpreter.get())
     if not self._interpreter.satisfies(self.mixins):
       raise self.IncapableInterpreter('Interpreter %s not capable of running %s' % (
           self._interpreter.binary, self.__class__.__name__))

pex/testing.py

@@ -13,7 +13,6 @@
 from collections import namedtuple
 from textwrap import dedent
 
-from pex import vendor
 from pex.bin import pex as pex_bin_pex
 from pex.common import open_zip, safe_mkdir, safe_rmtree, touch
 from pex.compatibility import PY3, nested
@@ -140,7 +139,6 @@ def make_installer(name='my_project',
             'zip_safe': zip_safe,
             'install_requires': install_reqs or []}
   with temporary_content(PROJECT_CONTENT, interp=interp) as td:
-    interpreter = vendor.setup_interpreter(interpreter=interpreter)
     yield installer_impl(td, interpreter=interpreter, **kwargs)
 
 
@@ -210,7 +208,7 @@ def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False,
 
   pb = PEXBuilder(path=td,
                   preamble=COVERAGE_PREAMBLE if coverage else None,
-                  interpreter=vendor.setup_interpreter(interpreter=interpreter))
+                  interpreter=interpreter)
 
   for dist in dists:
     pb.add_dist_location(dist.location)
@@ -285,7 +283,7 @@ def update_env(target_env):
 
 
 def run_simple_pex(pex, args=(), interpreter=None, stdin=None, **kwargs):
-  p = PEX(pex, interpreter=vendor.setup_interpreter(interpreter))
+  p = PEX(pex, interpreter=interpreter)
   process = p.run(args=args,
                   blocking=False,
                   stdin=subprocess.PIPE,
@@ -299,7 +297,6 @@ def run_simple_pex(pex, args=(), interpreter=None, stdin=None, **kwargs):
 
 def run_simple_pex_test(body, args=(), env=None, dists=None, coverage=False, interpreter=None):
   with nested(temporary_dir(), temporary_dir()) as (td1, td2):
-    interpreter = vendor.setup_interpreter(interpreter=interpreter)
     pb = write_simple_pex(td1, body, dists=dists, coverage=coverage, interpreter=interpreter)
     pex = os.path.join(td2, 'app.pex')
     pb.build(pex)

pex/vendor/__init__.py

@@ -7,6 +7,7 @@
 import os
 
 from pex.common import touch
+from pex.orderedset import OrderedSet
 from pex.tracer import TRACER
 
 
@@ -71,41 +72,43 @@ def create_packages(self):
       touch(os.path.join(self.ROOT, *relpath))
 
 
-def iter_vendor_specs(include_wheel=True):
+def iter_vendor_specs():
   """Iterate specifications for code vendored by pex.
 
-  :param bool include_wheel: If ``True`` include the vendored wheel spec.
-  :return: An iterator over specs of all vendored code optionally including ``wheel``.
+  :return: An iterator over specs of all vendored code.
   :rtype: :class:`collection.Iterator` of :class:`VendorSpec`
   """
   yield VendorSpec.create('setuptools==40.6.2')
-  if include_wheel:
-    # We're currently stuck here due to removal of an API we depend on.
-    # See: https://github.com/pantsbuild/pex/issues/603
-    yield VendorSpec.create('wheel==0.31.1')
 
+  # We're currently stuck here due to removal of an API we depend on.
+  # See: https://github.com/pantsbuild/pex/issues/603
+  yield VendorSpec.create('wheel==0.31.1')
 
-def _vendored_dists(include_wheel=True):
-  entries = [spec.target_dir for spec in iter_vendor_specs(include_wheel=include_wheel)]
+
+def _vendored_dists(distributions):
+  entries = [spec.target_dir for spec in iter_vendor_specs() if spec.key in distributions]
 
   import pex.third_party.pkg_resources as pkg_resources
   return list(pkg_resources.WorkingSet(entries=entries))
 
 
-def setup_interpreter(interpreter=None, include_wheel=True):
+def setup_interpreter(distributions, interpreter=None):
   """Return an interpreter configured with vendored distributions as extras.
 
-  :param interpreter: An option interpreter to configure. If ``None``, the current interpreter is
+  Any distributions that are present in the vendored set will be added to the interpreter as extras.
+
+  :param distributions: The names of distributions to setup the interpreter with.
+  :type distributions: list of str
+  :param interpreter: An optional interpreter to configure. If ``None``, the current interpreter is
                       used.
   :type interpreter: :class:`pex.interpreter.PythonInterpreter`
-  :param bool include_wheel: If ``True`` include the vendored wheel distribution.
   :return: An bare interpreter configured with vendored extras.
   :rtype: :class:`pex.interpreter.PythonInterpreter`
   """
   from pex.interpreter import PythonInterpreter
 
   interpreter = interpreter or PythonInterpreter.get()
-  for dist in _vendored_dists(include_wheel=include_wheel):
+  for dist in _vendored_dists(OrderedSet(distributions)):
     interpreter = interpreter.with_extra(dist.key, dist.version, dist.location)
   return interpreter
 

tests/test_bdist_pex.py

@@ -7,9 +7,9 @@
 from textwrap import dedent
 
 import pex.third_party.pkg_resources as pkg_resources
-from pex import vendor
 from pex.common import open_zip
 from pex.installer import DistributionPackager, WheelInstaller, after_installation
+from pex.interpreter import PythonInterpreter
 from pex.testing import temporary_content
 
 
@@ -31,9 +31,9 @@ def bdist(self):
     return self.find_distribution()
 
 
-def bdist_pex_installer(source_dir, interpreter=None, bdist_args=None):
-  interpreter = vendor.setup_interpreter(interpreter=interpreter)
+def bdist_pex_installer(source_dir, bdist_args=None):
   pex_dist = pkg_resources.working_set.find(pkg_resources.Requirement.parse('pex'))
+  interpreter = PythonInterpreter.get()
   interpreter = interpreter.with_extra(pex_dist.key, pex_dist.version, pex_dist.location)
   return BdistPexInstaller(source_dir=source_dir, interpreter=interpreter, bdist_args=bdist_args)
 
@@ -118,7 +118,7 @@ def test_unwriteable_contents():
                           'my_app/__init__.py': '',
                           'my_app/unwriteable.so': ''},
                          perms=UNWRITEABLE_PERMS) as my_app_project_dir:
-    my_app_whl = WheelInstaller(my_app_project_dir, vendor.setup_interpreter()).bdist()
+    my_app_whl = WheelInstaller(my_app_project_dir).bdist()
 
     uses_my_app_setup_py = bdist_pex_setup_py(name='uses_my_app',
                                               version='0.0.0',

tests/test_environment.py

@@ -9,7 +9,7 @@
 
 import pytest
 
-from pex import resolver, vendor
+from pex import resolver
 from pex.compatibility import PY2, nested, to_bytes
 from pex.environment import PEXEnvironment
 from pex.installer import EggInstaller, WheelInstaller
@@ -31,7 +31,6 @@
 
 @contextmanager
 def yield_pex_builder(zip_safe=True, installer_impl=EggInstaller, interpreter=None):
-  interpreter = vendor.setup_interpreter(interpreter=interpreter)
   with nested(temporary_dir(),
               make_bdist('p1',
                          zipped=True,
@@ -139,7 +138,6 @@ def add_sources(builder, content):
       for path in content.keys():
         builder.add_source(os.path.join(project, path), path)
 
-  interpreter = vendor.setup_interpreter(interpreter=interpreter)
   with nested(temporary_dir(), temporary_dir()) as (root, cache):
     pex_info1 = PexInfo.default()
     pex_info1.zip_safe = False

tests/test_inherits_path_option.py

@@ -4,7 +4,6 @@
 import os
 from contextlib import contextmanager
 
-from pex import vendor
 from pex.pex_builder import PEXBuilder
 from pex.testing import run_simple_pex, temporary_dir
 
@@ -21,7 +20,7 @@ def write_and_run_simple_pex(inheriting=False):
     with open(os.path.join(td, 'exe.py'), 'w') as fp:
       fp.write('')  # No contents, we just want the startup messages
 
-    pb = PEXBuilder(path=td, preamble=None, interpreter=vendor.setup_interpreter())
+    pb = PEXBuilder(path=td, preamble=None)
     pb.info.inherit_path = inheriting
     pb.set_executable(os.path.join(td, 'exe.py'))
     pb.freeze()

tests/test_integration.py

@@ -11,7 +11,6 @@
 
 import pytest
 
-from pex import vendor
 from pex.compatibility import WINDOWS, nested, to_bytes
 from pex.installer import EggInstaller
 from pex.pex_bootstrapper import get_pex_info
@@ -162,7 +161,7 @@ def do_something():
   """ % error_msg)
 
   with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir:
-    installer = EggInstaller(project_dir, interpreter=vendor.setup_interpreter())
+    installer = EggInstaller(project_dir)
     dist = DistributionHelper.distribution_from_path(installer.bdist())
     so, rc = run_simple_pex_test('', env=make_env(PEX_SCRIPT='my_app'), dists=[dist])
     assert so.decode('utf-8').strip() == error_msg

tests/test_pex.py

@@ -11,7 +11,6 @@
 
 import pytest
 
-from pex import vendor
 from pex.compatibility import PY2, WINDOWS, nested, to_bytes
 from pex.installer import EggInstaller, WheelInstaller
 from pex.interpreter import PythonInterpreter
@@ -321,9 +320,7 @@ def test_pex_verify_entry_point_module_should_fail():
 def test_activate_interpreter_different_from_current():
   with temporary_dir() as pex_root:
     interp_version = PY36 if PY2 else PY27
-    custom_interpreter = vendor.setup_interpreter(
-      interpreter=PythonInterpreter.from_binary(ensure_python_interpreter(interp_version)),
-    )
+    custom_interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(interp_version))
     pex_info = PexInfo.default(custom_interpreter)
     pex_info.pex_root = pex_root
     with temporary_dir() as pex_chroot:
@@ -359,15 +356,14 @@ def test_execute_interpreter_dashc_program():
 
 def test_execute_interpreter_dashm_module():
   with temporary_dir() as pex_chroot:
-    interpreter = vendor.setup_interpreter()
-    pex_builder = PEXBuilder(path=pex_chroot, interpreter=interpreter)
+    pex_builder = PEXBuilder(path=pex_chroot)
     pex_builder.add_source(None, 'foo/__init__.py')
     with tempfile.NamedTemporaryFile() as fp:
       fp.write(b'import sys; print(" ".join(sys.argv))')
       fp.flush()
       pex_builder.add_source(fp.name, 'foo/bar.py')
     pex_builder.freeze()
-    pex = PEX(pex_chroot, interpreter=vendor.setup_interpreter())
+    pex = PEX(pex_chroot)
     process = pex.run(args=['-m', 'foo.bar', 'one', 'two'],
                       stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE,

tests/test_pex_builder.py

@@ -6,7 +6,6 @@
 
 import pytest
 
-from pex import vendor
 from pex.common import open_zip
 from pex.compatibility import WINDOWS, nested
 from pex.pex import PEX
@@ -79,13 +78,9 @@ def test_pex_builder_wheeldep():
       assert fp.read() == 'success'
 
 
-def pex_builder(interpreter=None, **kwargs):
-  return PEXBuilder(interpreter=vendor.setup_interpreter(interpreter), **kwargs)
-
-
 def test_pex_builder_shebang():
   def builder(shebang):
-    pb = pex_builder()
+    pb = PEXBuilder()
     pb.set_shebang(shebang)
     return pb
 
@@ -110,7 +105,7 @@ def test_pex_builder_preamble():
       "sys.exit(3)"
     ])
 
-    pb = pex_builder(preamble=tempfile_preamble)
+    pb = PEXBuilder(preamble=tempfile_preamble)
     pb.build(target)
 
     assert not os.path.exists(should_create)
@@ -134,7 +129,7 @@ def test_pex_builder_compilation():
       fp.write(exe_main)
 
     def build_and_check(path, precompile):
-      pb = pex_builder(path=path)
+      pb = PEXBuilder(path=path)
       pb.add_source(src, 'lib/src.py')
       pb.set_executable(exe, 'exe.py')
       pb.freeze(bytecode_compile=precompile)
@@ -165,7 +160,7 @@ def test_pex_builder_copy_or_link():
       fp.write(exe_main)
 
     def build_and_check(path, copy):
-      pb = pex_builder(path=path, copy=copy)
+      pb = PEXBuilder(path=path, copy=copy)
       pb.add_source(src, 'exe.py')
 
       path_clone = os.path.join(path, '__clone')

tests/test_resolvable.py

@@ -4,7 +4,7 @@
 import pytest
 
 import pex.third_party.pkg_resources as pkg_resources
-from pex import vendor
+from pex.interpreter import PythonInterpreter
 from pex.iterator import Iterator
 from pex.package import Package, SourcePackage
 from pex.resolvable import (
@@ -85,7 +85,7 @@ def test_resolvable_requirement():
 
 def test_resolvable_directory():
   builder = ResolverOptionsBuilder()
-  interpreter = vendor.setup_interpreter()
+  interpreter = PythonInterpreter.get()
 
   with make_source_dir(name='my_project') as td:
     rdir = ResolvableDirectory.from_string(td, builder, interpreter)

tests/test_resolver.py

@@ -6,7 +6,6 @@
 
 import pytest
 
-from pex import vendor
 from pex.common import safe_copy
 from pex.crawler import Crawler
 from pex.fetcher import Fetcher
@@ -18,7 +17,6 @@
 
 
 def do_resolve_multi(*args, **kwargs):
-  kwargs.setdefault('interpreters', [vendor.setup_interpreter()])
   return list(resolve_multi(*args, **kwargs))
 
 

tests/test_util.py

@@ -7,7 +7,6 @@
 from hashlib import sha1
 from textwrap import dedent
 
-from pex import vendor
 from pex.common import open_zip, safe_mkdir
 from pex.compatibility import nested, to_bytes
 from pex.installer import EggInstaller, WheelInstaller
@@ -136,7 +135,7 @@ def assert_access_zipped_assets(distribution_helper_import):
           print(line)
   """.format(distribution_helper_import=distribution_helper_import))
   with nested(temporary_dir(), temporary_dir()) as (td1, td2):
-    pb = PEXBuilder(path=td1, interpreter=vendor.setup_interpreter())
+    pb = PEXBuilder(path=td1)
     with open(os.path.join(td1, 'exe.py'), 'w') as fp:
       fp.write(test_executable)
       pb.set_executable(fp.name)