Upgrade to pex 1.4.8 and eliminate workarounds. (#6594)

Pex now allows us to float setuptools high and run on python 3.7 and
it supports sane platform expansion as well as graceful handling of
non-standard `setuptools` platform reporting for Apple system
interpreters allowing us to eliminate several workarounds.

We also pickup a fix for concurrent pex extraction which is particularly
useful when running pants from a pex using `PEX_FORCE_LOCAL`.

Fixes #5922
Fixes #6363

.travis.yml

@@ -241,12 +241,6 @@ matrix:
       osx_image: xcode8.3
       stage: Test Pants
       language: generic
-      env:
-        # Specifically avoid the OSX provided 2.7.10 under xcode8.3 since it returns a platform
-        # of `macosx-*-intel` where the `intel` suffix is bogus but pex has not yet been taught to
-        # deal with this. Can be removed when this issue is resolved:
-        # https://github.com/pantsbuild/pex/issues/523
-        - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython>2.7.10,<3']"
       before_install:
         - brew tap caskroom/cask && brew update && brew cask install osxfuse
       before_script:

3rdparty/python/requirements.txt

@@ -14,7 +14,7 @@ mock==2.0.0
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.0
-pex==1.4.5
+pex==1.4.8
 psutil==4.3.0
 pycodestyle==2.4.0
 pyflakes==2.0.0
@@ -27,9 +27,9 @@ pywatchman==1.4.1
 requests[security]>=2.5.0,<2.19
 scandir==1.2
 setproctitle==1.1.10
-setuptools==33.1.1
+setuptools==40.4.3
 six>=1.9.0,<2
 subprocess32==3.2.7 ; python_version<'3'
 thrift>=0.9.1
-wheel==0.29.0
+wheel==0.31.1
 www-authenticate==0.9.2

build-support/bin/ci.sh

@@ -130,9 +130,7 @@ fi
 if [[ "${python_three:-false}" == "true" ]]; then
   # The 3.4 end of this constraint is necessary to jive with the travis ubuntu trusty image.
   banner "Setting interpreter constraints for 3!"
-  # TODO(John Sirois): Allow `<4` when the issues with `3.7` are fixed. See:
-  #   https://github.com/pantsbuild/pants/issues/6363
-  export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<3.7"]'
+  export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.4,<4"]'
   # TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache
   # bug between the `./pants binary` and further runs.
   ./pants.pex clean-all

src/python/pants/backend/python/BUILD

@@ -41,7 +41,6 @@ python_library(
   dependencies = [
     '3rdparty/python:future',
     '3rdparty/python:pex',
-    ':pex_util',
     'src/python/pants/backend/python/targets',
     'src/python/pants/base:exceptions',
     'src/python/pants/process',
@@ -88,10 +87,3 @@ python_library(
   ]
 )
 
-python_library(
-  name = 'pex_util',
-  sources = ['pex_util.py'],
-  dependencies = [
-    '3rdparty/python:pex',
-  ]
-)

src/python/pants/backend/python/interpreter_cache.py

@@ -14,7 +14,6 @@
 from pex.resolver import resolve
 from pex.variables import Variables
 
-from pants.backend.python.pex_util import expand_and_maybe_adjust_platform
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.exceptions import TaskError
 from pants.process.lock import OwnerPrintingInterProcessFileLock
@@ -225,11 +224,9 @@ def _resolve_and_link(self, interpreter, requirement, target_link):
     distributions = resolve(requirements=[requirement],
                             fetchers=self._python_repos.get_fetchers(),
                             interpreter=interpreter,
-                            platform=expand_and_maybe_adjust_platform(
-                              interpreter=interpreter,
-                              # The local interpreter cache is, by definition, composed of
-                              # interpreters for the 'current' platform.
-                              platform='current'),
+                            # The local interpreter cache is, by definition, composed of
+                            # interpreters for the 'current' platform.
+                            platform='current',
                             context=self._python_repos.get_network_context(),
                             precedence=precedence)
     if not distributions:

src/python/pants/backend/python/subsystems/python_setup.py

@@ -26,9 +26,9 @@ def register_options(cls, register):
                   "or 'PyPy' (A pypy interpreter of any version). Multiple constraint strings will "
                   "be ORed together. These constraints are applied in addition to any "
                   "compatibilities required by the relevant targets.")
-    register('--setuptools-version', advanced=True, default='33.1.1',
+    register('--setuptools-version', advanced=True, default='40.4.3',
              help='The setuptools version for this python environment.')
-    register('--wheel-version', advanced=True, default='0.29.0',
+    register('--wheel-version', advanced=True, default='0.31.1',
              help='The wheel version for this python environment.')
     register('--platforms', advanced=True, type=list, metavar='<platform>', default=['current'],
              help='A list of platforms to be supported by this python environment. Each platform'

src/python/pants/backend/python/tasks/BUILD

@@ -13,7 +13,6 @@ python_library(
     'src/python/pants/backend/native/targets',
     'src/python/pants/backend/native/tasks',
     'src/python/pants/backend/python:interpreter_cache',
-    'src/python/pants/backend/python:pex_util',
     'src/python/pants/backend/python:python_requirement',
     'src/python/pants/backend/python/subsystems',
     'src/python/pants/backend/python/targets',

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -12,7 +12,6 @@
 from pex.resolver import resolve
 from twitter.common.collections import OrderedSet
 
-from pants.backend.python.pex_util import expand_and_maybe_adjust_platform
 from pants.backend.python.subsystems.python_repos import PythonRepos
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_binary import PythonBinary
@@ -160,7 +159,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links):
       requirements=[req.requirement for req in requirements],
       interpreter=interpreter,
       fetchers=fetchers,
-      platform=expand_and_maybe_adjust_platform(interpreter=interpreter, platform=platform),
+      platform=platform,
       context=python_repos.get_network_context(),
       cache=requirements_cache_dir,
       cache_ttl=python_setup.resolver_cache_ttl,

src/python/pants/backend/python/tasks/pytest_prep.py

@@ -8,6 +8,9 @@
 from builtins import object
 
 import pkg_resources
+from pex.interpreter import PythonInterpreter
+from pex.pex import PEX
+from pex.pex_builder import PEXBuilder
 from pex.pex_info import PexInfo
 
 from pants.backend.python.subsystems.pytest import PyTest
@@ -23,8 +26,15 @@ class PytestBinary(object):
 
     _COVERAGE_PLUGIN_MODULE_NAME = '__{}__'.format(__name__.replace('.', '_'))
 
-    def __init__(self, pex):
-      self._pex = pex
+    def __init__(self, interpreter, pex):
+      # Here we hack around `coverage.cmdline` nuking the 0th element of `sys.path` (our root pex)
+      # by ensuring, the root pex is on the sys.path twice.
+      # See: https://github.com/nedbat/coveragepy/issues/715
+      pex_path = pex.path()
+      pex_info = PexInfo.from_pex(pex_path)
+      pex_info.merge_pex_path(pex_path)  # We're now on the sys.path twice.
+      PEXBuilder(pex_path, interpreter=interpreter, pex_info=pex_info).freeze()
+      self._pex = PEX(pex=pex_path, interpreter=interpreter)
 
     @property
     def pex(self):
@@ -76,4 +86,6 @@ def execute(self):
     pex_info = PexInfo.default()
     pex_info.entry_point = 'pytest'
     pytest_binary = self.create_pex(pex_info)
-    self.context.products.register_data(self.PytestBinary, self.PytestBinary(pytest_binary))
+    interpreter = self.context.products.get_data(PythonInterpreter)
+    self.context.products.register_data(self.PytestBinary,
+                                        self.PytestBinary(interpreter, pytest_binary))