Add blacklist handling for skipping requirements in pex resolver (#457)

Problem
When resolving for a specific interpreter version, the pex resolver fails to resolve universal requirements that transitively require libraries with interpreter constraints that conflict with the target interpreter version (see: https://github.com/Julian/jsonschema/blob/master/setup.py#L43 ).

Solution
The short term solution is to allow the caller of pex.resolver#resolve to supply a blacklist dict that maps package name -> interpreter constraint, enabling the resolver to skip blacklisted requirement names at resolve time if the target interpreter conforms with the interpreter constraint. The long-term solution is to be addressed by #456.

Result
It is now possible for systems that build pex files (Pants) via the pex api to blacklist certain requirements that are redundant (backports) or otherwise should not be included in the output pex.

pex/resolver.py

@@ -151,10 +151,11 @@ def filter_packages_by_interpreter(cls, packages, interpreter, platform):
     return [package for package in packages
         if package.compatible(interpreter.identity, platform)]
 
-  def __init__(self, allow_prereleases=None, interpreter=None, platform=None):
+  def __init__(self, allow_prereleases=None, interpreter=None, platform=None, pkg_blacklist=None):
     self._interpreter = interpreter or PythonInterpreter.get()
     self._platform = platform or Platform.current()
     self._allow_prereleases = allow_prereleases
+    self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {}
 
   def package_iterator(self, resolvable, existing=None):
     if existing:
@@ -180,6 +181,12 @@ def build(self, package, options):
         'Could not get distribution for %s on platform %s.' % (package, self._platform))
     return dist
 
+  def _resolvable_is_blacklisted(self, resolvable_name):
+    return (
+      resolvable_name in self._blacklist and
+      self._interpreter.identity.matches(self._blacklist[resolvable_name])
+    )
+
   def resolve(self, resolvables, resolvable_set=None):
     resolvables = [(resolvable, None) for resolvable in resolvables]
     resolvable_set = resolvable_set or _ResolvableSet()
@@ -193,7 +200,11 @@ def resolve(self, resolvables, resolvable_set=None):
         if resolvable in processed_resolvables:
           continue
         packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))
-        resolvable_set.merge(resolvable, packages, parent)
+
+        # TODO: Remove blacklist strategy in favor of smart requirement handling
+        # https://github.com/pantsbuild/pex/issues/456
+        if not self._resolvable_is_blacklisted(resolvable.name):
+          resolvable_set.merge(resolvable, packages, parent)
         processed_resolvables.add(resolvable)
 
       built_packages = {}
@@ -299,7 +310,8 @@ def resolve(requirements,
             precedence=None,
             cache=None,
             cache_ttl=None,
-            allow_prereleases=None):
+            allow_prereleases=None,
+            pkg_blacklist=None):
   """Produce all distributions needed to (recursively) meet `requirements`
 
   :param requirements: An iterator of Requirement-like things, either
@@ -329,6 +341,14 @@ def resolve(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
+  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
+    an interpreter constraint. If a package name is in the blacklist and its interpreter
+    constraint matches the target interpreter, skip the requirement. This is needed to ensure
+    that universal requirement resolves for a target interpreter version do not error out on
+    interpreter specific requirements such as backport libs like `functools32`.
+    For example, a valid blacklist is {'functools32': 'CPython>3'}.
+    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
+    tracked by: https://github.com/pantsbuild/pex/issues/456
   :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
@@ -367,11 +387,13 @@ def resolve(requirements,
                                cache_ttl,
                                allow_prereleases=allow_prereleases,
                                interpreter=interpreter,
-                               platform=platform)
+                               platform=platform,
+                               pkg_blacklist=pkg_blacklist)
   else:
     resolver = Resolver(allow_prereleases=allow_prereleases,
                         interpreter=interpreter,
-                        platform=platform)
+                        platform=platform,
+                        pkg_blacklist=pkg_blacklist)
 
   return resolver.resolve(resolvables_from_iterable(requirements, builder))
 
@@ -384,7 +406,8 @@ def resolve_multi(requirements,
                   precedence=None,
                   cache=None,
                   cache_ttl=None,
-                  allow_prereleases=None):
+                  allow_prereleases=None,
+                  pkg_blacklist=None):
   """A generator function that produces all distributions needed to meet `requirements`
   for multiple interpreters and/or platforms.
 
@@ -415,6 +438,14 @@ def resolve_multi(requirements,
     ``context``.
   :keyword allow_prereleases: (optional) Include pre-release and development versions.  If
     unspecified only stable versions will be resolved, unless explicitly included.
+  :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
+    an interpreter constraint. If a package name is in the blacklist and its interpreter
+    constraint matches the target interpreter, skip the requirement. This is needed to ensure
+    that universal requirement resolves for a target interpreter version do not error out on
+    interpreter specific requirements such as backport libs like `functools32`.
+    For example, a valid blacklist is {'functools32': 'CPython>3'}.
+    NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution
+    tracked by: https://github.com/pantsbuild/pex/issues/456
   :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``.
   :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
   :raises Untranslateable: If no compatible distributions could be acquired for
@@ -435,7 +466,8 @@ def resolve_multi(requirements,
                                 precedence,
                                 cache,
                                 cache_ttl,
-                                allow_prereleases):
+                                allow_prereleases,
+                                pkg_blacklist=pkg_blacklist):
         if resolvable not in seen:
           seen.add(resolvable)
           yield resolvable

tests/test_resolver.py

@@ -2,17 +2,20 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
+import sys
 import time
 
 import pytest
 from twitter.common.contextutil import temporary_dir
 
 from pex.common import safe_copy
+from pex.compatibility import PY2
 from pex.crawler import Crawler
 from pex.fetcher import Fetcher
+from pex.interpreter import PythonInterpreter
 from pex.package import EggPackage, SourcePackage
 from pex.resolvable import ResolvableRequirement
-from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi
+from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve, resolve_multi
 from pex.resolver_options import ResolverOptionsBuilder
 from pex.testing import make_sdist
 
@@ -349,3 +352,26 @@ def test_resolvable_set_built():
   updated_rs.merge(rq, [binary_pkg])
   assert updated_rs.get('foo') == set([binary_pkg])
   assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)]
+
+
+def test_resolver_blacklist():
+  if PY2:
+    blacklist = {'project2': '<3'}
+    required_project = "project2;python_version>'3'"
+  else:
+    blacklist = {'project2': '>3'}
+    required_project = "project2;python_version<'3'"
+
+  project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project])
+  project2 = make_sdist(name='project2', version='1.1.0')
+
+  with temporary_dir() as td:
+    safe_copy(project1, os.path.join(td, os.path.basename(project1)))
+    safe_copy(project2, os.path.join(td, os.path.basename(project2)))
+    fetchers = [Fetcher([td])]
+
+    dists = resolve(['project1'], fetchers=fetchers)
+    assert len(dists) == 2
+
+    dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist)
+    assert len(dists) == 1