Vendor setuptools and wheel.

Fixes pex-tool#607

MANIFEST.in

@@ -2,4 +2,5 @@ include *.py *.rst *.ini MANIFEST.in LICENSE
 recursive-include docs *
 recursive-include tests *
 recursive-include scripts *
+recursive-include pex/vendor/_vendored *
 recursive-exclude * *.pyc *~

pex/bin/pex.py

@@ -2,41 +2,33 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 """
-The pex.pex utility builds PEX environments and .pex files specified by
+The pex.bin.pex utility builds PEX environments and .pex files specified by
 sources, requirements and their dependencies.
 """
 
 from __future__ import absolute_import, print_function
 
-import functools
 import os
-import shutil
 import sys
 from optparse import OptionGroup, OptionParser, OptionValueError
 from textwrap import TextWrapper
 
-from pex.archiver import Archiver
-from pex.base import maybe_requirement
-from pex.common import die, safe_delete, safe_mkdir, safe_mkdtemp
-from pex.crawler import Crawler
+from pex import vendor
+from pex.common import die, safe_delete, safe_mkdtemp
 from pex.fetcher import Fetcher, PyPIFetcher
-from pex.http import Context
-from pex.installer import EggInstaller
 from pex.interpreter import PythonInterpreter
 from pex.interpreter_constraints import validate_constraints
-from pex.iterator import Iterator
-from pex.package import EggPackage, SourcePackage
 from pex.pex import PEX
 from pex.pex_bootstrapper import find_compatible_interpreters
 from pex.pex_builder import PEXBuilder
 from pex.platforms import Platform
 from pex.requirements import requirements_from_file
-from pex.resolvable import Resolvable
+from pex.resolvable import resolvables_from_iterable
 from pex.resolver import Unsatisfiable, resolve_multi
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.tracer import TRACER
 from pex.variables import ENV, Variables
-from pex.version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT, __version__
+from pex.version import __version__
 
 CANNOT_DISTILL = 101
 CANNOT_SETUP_INTERPRETER = 102
@@ -499,78 +491,6 @@ def _safe_link(src, dst):
   os.symlink(src, dst)
 
 
-def _resolve_and_link_interpreter(requirement, fetchers, target_link, installer_provider):
-  # Short-circuit if there is a local copy
-  if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
-    egg = EggPackage(os.path.realpath(target_link))
-    if egg.satisfies(requirement):
-      return egg
-
-  context = Context.get()
-  iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
-  links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)]
-
-  with TRACER.timed('Interpreter cache resolving %s' % requirement, V=2):
-    for link in links:
-      with TRACER.timed('Fetching %s' % link, V=3):
-        sdist = context.fetch(link)
-
-      with TRACER.timed('Installing %s' % link, V=3):
-        installer = installer_provider(sdist)
-        dist_location = installer.bdist()
-        target_location = os.path.join(
-            os.path.dirname(target_link), os.path.basename(dist_location))
-        shutil.move(dist_location, target_location)
-        _safe_link(target_location, target_link)
-
-      return EggPackage(target_location)
-
-
-def resolve_interpreter(cache, fetchers, interpreter, requirement):
-  """Resolve an interpreter with a specific requirement.
-
-     Given a :class:`PythonInterpreter` and a requirement, return an
-     interpreter with the capability of resolving that requirement or
-     ``None`` if it's not possible to install a suitable requirement."""
-  requirement = maybe_requirement(requirement)
-
-  # short circuit
-  if interpreter.satisfies([requirement]):
-    return interpreter
-
-  def installer_provider(sdist):
-    return EggInstaller(
-        Archiver.unpack(sdist),
-        strict=requirement.key != 'setuptools',
-        interpreter=interpreter)
-
-  interpreter_dir = os.path.join(cache, str(interpreter.identity))
-  safe_mkdir(interpreter_dir)
-
-  egg = _resolve_and_link_interpreter(
-      requirement,
-      fetchers,
-      os.path.join(interpreter_dir, requirement.key),
-      installer_provider)
-
-  if egg:
-    return interpreter.with_extra(egg.name, egg.raw_version, egg.path)
-
-
-def setup_interpreter(interpreter, interpreter_cache_dir, repos, use_wheel):
-  with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
-    resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos)
-
-    # resolve setuptools
-    interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
-
-    # possibly resolve wheel
-    if interpreter and use_wheel:
-      interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
-
-    return interpreter
-
-
 def build_pex(args, options, resolver_option_builder):
   with TRACER.timed('Resolving interpreters', V=2):
     def to_python_interpreter(full_path_or_basename):
@@ -593,10 +513,7 @@ def to_python_interpreter(full_path_or_basename):
     pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '')
     interpreters = find_compatible_interpreters(pex_python_path, constraints)
 
-  setup_interpreters = [setup_interpreter(interp,
-                                          options.interpreter_cache_dir,
-                                          options.repos,
-                                          options.use_wheel)
+  setup_interpreters = [vendor.setup_interpreter(interpreter=interp, use_wheel=options.use_wheel)
                         for interp in interpreters]
 
   if not setup_interpreters:
@@ -637,16 +554,20 @@ def walk_and_do(fn, src_dir):
     for ic in options.interpreter_constraint:
       pex_builder.add_interpreter_constraint(ic)
 
-  resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]
+  resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter)
 
   for requirements_txt in options.requirement_files:
-    resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))
+    resolvables.extend(requirements_from_file(requirements_txt,
+                                              builder=resolver_option_builder,
+                                              interpreter=interpreter))
 
   # pip states the constraints format is identical tor requirements
   # https://pip.pypa.io/en/stable/user_guide/#constraints-files
   for constraints_txt in options.constraint_files:
     constraints = []
-    for r in requirements_from_file(constraints_txt, resolver_option_builder):
+    for r in requirements_from_file(constraints_txt,
+                                    builder=resolver_option_builder,
+                                    interpreter=interpreter):
       r.is_constraint = True
       constraints.append(r)
     resolvables.extend(constraints)
@@ -720,6 +641,8 @@ def main(args=None):
   if options.python and options.interpreter_constraint:
     die('The "--python" and "--interpreter-constraint" options cannot be used together.')
 
+  vendor.adjust_sys_path(include_wheel=options.use_wheel)
+
   if options.pex_root:
     ENV.set('PEX_ROOT', options.pex_root)
   else:

pex/commands/bdist_pex.py

@@ -4,6 +4,7 @@
 
 from setuptools import Command
 
+from pex import vendor
 from pex.bin.pex import build_pex, configure_clp, make_relative_to_root
 from pex.common import die
 from pex.compatibility import ConfigParser, StringIO, string, to_unicode
@@ -86,7 +87,8 @@ def run(self):
     reqs = [package_dir] + reqs
 
     with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root):
-      pex_builder = build_pex(reqs, options, options_builder)
+      with vendor.adjusted_sys_path(include_wheel=True):
+        pex_builder = build_pex(reqs, options, options_builder)
 
     console_scripts = self.parse_entry_points()
 

pex/compiler.py

@@ -82,7 +82,7 @@ def compile(self, root, relpaths):
       fp.flush()
 
       try:
-        out, _ = Executor.execute([self._interpreter.binary, fp.name])
+        out, _ = Executor.execute([self._interpreter.binary, '-sE', fp.name])
       except Executor.NonZeroExit as e:
         raise self.CompilationFailure(
           'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr)

pex/environment.py

@@ -170,12 +170,8 @@ def _resolve(self, working_set, reqs):
           resolveds.update(working_set.resolve([req], env=self))
         except DistributionNotFound as e:
           TRACER.log('Failed to resolve a requirement: %s' % e)
-          unresolved_reqs.add(e.args[0].project_name)
-          # Older versions of pkg_resources just call `DistributionNotFound(req)` instead of the
-          # modern `DistributionNotFound(req, requirers)` and so we may not have the 2nd requirers
-          # slot at all.
-          if len(e.args) >= 2 and e.args[1]:
-            unresolved_reqs.update(e.args[1])
+          unresolved_reqs.add(e.req.project_name)
+          unresolved_reqs.update(e.requirers_str)
 
     unresolved_reqs = set([req.lower() for req in unresolved_reqs])
 

pex/executor.py

@@ -56,47 +56,37 @@ def __init__(self, cmd, exc):
       self.exc = exc
 
   @classmethod
-  def open_process(cls, cmd, env=None, cwd=None, combined=False, **kwargs):
+  def open_process(cls, cmd, **kwargs):
     """Opens a process object via subprocess.Popen().
 
     :param string|list cmd: A list or string representing the command to run.
-    :param dict env: An environment dict for the execution.
-    :param string cwd: The target cwd for command execution.
-    :param bool combined: Whether or not to combine stdin and stdout streams.
     :return: A `subprocess.Popen` object.
     :raises: `Executor.ExecutableNotFound` when the executable requested to run does not exist.
     """
     assert len(cmd) > 0, 'cannot execute an empty command!'
 
     try:
-      return subprocess.Popen(
-        cmd,
-        stdin=kwargs.pop('stdin', subprocess.PIPE),
-        stdout=kwargs.pop('stdout', subprocess.PIPE),
-        stderr=kwargs.pop('stderr', subprocess.STDOUT if combined else subprocess.PIPE),
-        cwd=cwd,
-        env=env,
-        **kwargs
-      )
+      return subprocess.Popen(cmd, **kwargs)
     except (IOError, OSError) as e:
       if e.errno == errno.ENOENT:
         raise cls.ExecutableNotFound(cmd, e)
       else:
         raise cls.ExecutionError(repr(e), cmd, e)
 
   @classmethod
-  def execute(cls, cmd, env=None, cwd=None, stdin_payload=None, **kwargs):
+  def execute(cls, cmd, stdin_payload=None, **kwargs):
     """Execute a command via subprocess.Popen and returns the stdio.
 
     :param string|list cmd: A list or string representing the command to run.
-    :param dict env: An environment dict for the execution.
-    :param string cwd: The target cwd for command execution.
     :param string stdin_payload: A string representing the stdin payload, if any, to send.
     :return: A tuple of strings representing (stdout, stderr), pre-decoded for utf-8.
     :raises: `Executor.ExecutableNotFound` when the executable requested to run does not exist.
              `Executor.NonZeroExit` when the execution fails with a non-zero exit code.
     """
-    process = cls.open_process(cmd=cmd, env=env, cwd=cwd, **kwargs)
+    process = cls.open_process(cmd=cmd,
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE, **kwargs)
     stdout_raw, stderr_raw = process.communicate(input=stdin_payload)
     # N.B. In cases where `stdout` or `stderr` is passed as parameters, these can be None.
     stdout = stdout_raw.decode('utf-8') if stdout_raw is not None else stdout_raw

pex/finders.py

@@ -60,8 +60,6 @@ def __eq__(self, other):
 # finders together.  This is probably possible using importlib but that does us no good as the
 # importlib machinery supporting this is only available in Python >= 3.1.
 def _get_finder(importer):
-  if not hasattr(pkg_resources, '_distribution_finders'):
-    return None
   return pkg_resources._distribution_finders.get(importer)
 
 
@@ -250,22 +248,6 @@ def register_finders():
   __PREVIOUS_FINDER = previous_finder
 
 
-def unregister_finders():
-  """Unregister finders necessary for PEX to function properly."""
-
-  global __PREVIOUS_FINDER
-  if not __PREVIOUS_FINDER:
-    return
-
-  pkg_resources.register_finder(zipimport.zipimporter, __PREVIOUS_FINDER)
-  _remove_finder(pkgutil.ImpImporter, find_wheels_on_path)
-
-  if importlib_machinery is not None:
-    _remove_finder(importlib_machinery.FileFinder, find_wheels_on_path)
-
-  __PREVIOUS_FINDER = None
-
-
 def get_script_from_egg(name, dist):
   """Returns location, content of script in distribution or (None, None) if not there."""
   if dist.metadata_isdir('scripts') and name in dist.metadata_listdir('scripts'):