Upgrade to pex 1.5.1; ~kill --resolver-blacklist. (#6619)

PEX now handles blacklisting for us by respecting PEP-508 enviornment
markers.

Fixes #5696

3rdparty/python/requirements.txt

@@ -14,7 +14,7 @@ mock==2.0.0
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.0
-pex==1.4.8
+pex==1.5.1
 psutil==4.3.0
 pycodestyle==2.4.0
 pyflakes==2.0.0

pants.ini

@@ -370,11 +370,6 @@ verify_commit: False
 interpreter_constraints: ["CPython>=2.7,<3"]
 interpreter_cache_dir: %(pants_bootstrapdir)s/python_cache/interpreters
 resolver_cache_dir: %(pants_bootstrapdir)s/python_cache/requirements
-resolver_blacklist: {
-    'subprocess32': 'CPython >= 3',
-    'faulthandler': 'CPython >= 3',
-    'futures': 'CPython >= 3',
-  }
 
 
 [test.pytest]

src/python/pants/backend/python/interpreter_cache.py

@@ -221,22 +221,22 @@ def _resolve_and_link(self, interpreter, requirement, target_link):
     # Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
     # wheels.
     precedence = (EggPackage, SourcePackage)
-    distributions = resolve(requirements=[requirement],
-                            fetchers=self._python_repos.get_fetchers(),
-                            interpreter=interpreter,
-                            # The local interpreter cache is, by definition, composed of
-                            # interpreters for the 'current' platform.
-                            platform='current',
-                            context=self._python_repos.get_network_context(),
-                            precedence=precedence)
-    if not distributions:
+    resolved_dists = resolve(requirements=[requirement],
+                             fetchers=self._python_repos.get_fetchers(),
+                             interpreter=interpreter,
+                             # The local interpreter cache is, by definition, composed of
+                             #  interpreters for the 'current' platform.
+                             platform='current',
+                             context=self._python_repos.get_network_context(),
+                             precedence=precedence)
+    if not resolved_dists:
       return None
 
-    assert len(distributions) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
+    assert len(resolved_dists) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
                                      'found:\n\t{}'.format(requirement,
-                                                           '\n\t'.join(map(str, distributions))))
+                                                           '\n\t'.join(map(str, resolved_dists))))
 
-    dist_location = distributions[0].location
+    dist_location = resolved_dists[0].distribution.location
     target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
     shutil.move(dist_location, target_location)
     _safe_link(target_location, target_link)

src/python/pants/backend/python/subsystems/python_setup.py

@@ -57,6 +57,11 @@ def register_options(cls, register):
               'variable is defined in a pexrc file, those interpreter paths will take precedence over ' 
               'this option.')
     register('--resolver-blacklist', advanced=True, type=dict, default={},
+             removal_version='1.13.0.dev2',
+             removal_hint='Now unused. Pants, via PEX, handles blacklisting automatically via '
+                          'PEP-508 environment markers anywhere Python requirements are specified '
+                          '(e.g. `requirements.txt` and `python_requirement(...)` in BUILD files): '
+                          'https://www.python.org/dev/peps/pep-0508/#environment-markers',
              metavar='<blacklist>',
              help='A blacklist dict (str->str) that maps package name to an interpreter '
               'constraint. If a package name is in the blacklist and its interpreter '
@@ -114,10 +119,6 @@ def resolver_cache_ttl(self):
   def resolver_allow_prereleases(self):
     return self.get_options().resolver_allow_prereleases
 
-  @property
-  def resolver_blacklist(self):
-    return self.get_options().resolver_blacklist
-
   @property
   def use_manylinux(self):
     return self.get_options().resolver_use_manylinux

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -112,11 +112,9 @@ def dump_requirements(builder, interpreter, reqs, log, platforms=None):
   """
   deduped_reqs = OrderedSet(reqs)
   find_links = OrderedSet()
-  blacklist = PythonSetup.global_instance().resolver_blacklist
   for req in deduped_reqs:
     log.debug('  Dumping requirement: {}'.format(req))
-    if not (req.key in blacklist and interpreter.identity.matches(blacklist[req.key])):
-      builder.add_requirement(req.requirement)
+    builder.add_requirement(req.requirement)
     if req.repository:
       find_links.add(req.repository)
 
@@ -155,7 +153,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links):
   for platform in platforms:
     requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
                                           str(interpreter.identity))
-    distributions[platform] = resolve(
+    resolved_dists = resolve(
       requirements=[req.requirement for req in requirements],
       interpreter=interpreter,
       fetchers=fetchers,
@@ -164,7 +162,7 @@ def resolve_multi(interpreter, requirements, platforms, find_links):
       cache=requirements_cache_dir,
       cache_ttl=python_setup.resolver_cache_ttl,
       allow_prereleases=python_setup.resolver_allow_prereleases,
-      pkg_blacklist=python_setup.resolver_blacklist,
       use_manylinux=python_setup.use_manylinux)
+    distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]
 
   return distributions

src/python/pants/init/plugin_resolver.py

@@ -108,15 +108,17 @@ def _resolve_exact_plugin_locations(self):
 
   def _resolve_plugins(self):
     logger.info('Resolving new plugins...:\n  {}'.format('\n  '.join(self._plugin_requirements)))
-    return resolver.resolve(self._plugin_requirements,
-                            fetchers=self._python_repos.get_fetchers(),
-                            context=self._python_repos.get_network_context(),
-                            cache=self.plugin_cache_dir,
-                            cache_ttl=10 * 365 * 24 * 60 * 60,  # Effectively never expire.
-                            allow_prereleases=PANTS_SEMVER.is_prerelease,
-                            # Plugins will all depend on `pantsbuild.pants` which is distributed as
-                            # a manylinux wheel.
-                            use_manylinux=True)
+    resolved_dists = resolver.resolve(self._plugin_requirements,
+                                      fetchers=self._python_repos.get_fetchers(),
+                                      context=self._python_repos.get_network_context(),
+                                      cache=self.plugin_cache_dir,
+                                      # Effectively never expire.
+                                      cache_ttl=10 * 365 * 24 * 60 * 60,
+                                      allow_prereleases=PANTS_SEMVER.is_prerelease,
+                                      # Plugins will all depend on `pantsbuild.pants` which is
+                                      # distributed as a manylinux wheel.
+                                      use_manylinux=True)
+    return [resolved_dist.distribution for resolved_dist in resolved_dists]
 
   @memoized_property
   def plugin_cache_dir(self):

tests/python/pants_test/backend/codegen/thrift/python/test_apache_thrift_py_gen.py

@@ -156,11 +156,11 @@ def test_namespace_effective(self):
     #   https://github.com/pantsbuild/pants/issues/5975
     pythonpath = list(interpreter.extras.values())
     pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
-    for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
-                        interpreter=interpreter,
-                        context=python_repos.get_network_context(),
-                        fetchers=python_repos.get_fetchers()):
-      pythonpath.append(dist.location)
+    for resolved_dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
+                                 interpreter=interpreter,
+                                 context=python_repos.get_network_context(),
+                                 fetchers=python_repos.get_fetchers()):
+      pythonpath.append(resolved_dist.distribution.location)
 
     process = subprocess.Popen([interpreter.binary,
                                 '-c',

tests/python/pants_test/backend/python/tasks/test_python_run_integration.py

@@ -13,8 +13,7 @@
                                                                    python_interpreter_path,
                                                                    skip_unless_python3,
                                                                    skip_unless_python27,
-                                                                   skip_unless_python27_and_python3,
-                                                                   skip_unless_python36)
+                                                                   skip_unless_python27_and_python3)
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_daemon
 from pants_test.testutils.pexrc_util import setup_pexrc_with_pex_python_path
 
@@ -200,29 +199,3 @@ def test_target_constraints_with_no_sources(self):
     self.assertIn('CPython>3', py2_info.interpreter_constraints)
     # Cleanup.
     os.remove(py2_pex)
-
-  @skip_unless_python36
-  def test_pex_resolver_blacklist_integration(self):
-    pex = os.path.join(os.getcwd(), 'dist', 'test_bin.pex')
-    try:
-      pants_ini_config = {'python-setup': {'resolver_blacklist': {'functools32': 'CPython>3'}}}
-      target_address_base = os.path.join(self.testproject, 'resolver_blacklist_testing')
-      # clean-all to ensure that Pants resolves requirements for each run.
-      pants_binary_36 = self.run_pants(
-        command=['clean-all', 'binary', '{}:test_bin'.format(target_address_base)],
-        config=pants_ini_config
-      )
-      self.assert_success(pants_binary_36)
-      pants_run_36 = self.run_pants(
-        command=['clean-all', 'run', '{}:test_bin'.format(target_address_base)],
-        config=pants_ini_config
-      )
-      self.assert_success(pants_run_36)
-      pants_run_27 = self.run_pants(
-        command=['clean-all', 'run', '{}:test_py2'.format(target_address_base)],
-        config=pants_ini_config
-      )
-      self.assert_success(pants_run_27)
-    finally:
-      if os.path.exists(pex):
-        os.remove(pex)

tests/python/pants_test/backend/python/test_interpreter_cache.py

@@ -85,11 +85,11 @@ def link_egg(repo_root, requirement):
         existing_dist = Package.from_href(existing_dist_location)
         requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)
 
-      distributions = resolve([requirement],
+      resolved_dists = resolve([requirement],
                               interpreter=self._interpreter,
                               precedence=(EggPackage, SourcePackage))
-      self.assertEqual(1, len(distributions))
-      dist_location = distributions[0].location
+      self.assertEqual(1, len(resolved_dists))
+      dist_location = resolved_dists[0].distribution.location
 
       self.assertRegexpMatches(dist_location, r'\.egg$')
       os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))