diff --git a/pex/resolver.py b/pex/resolver.py index 31a8e7e29..103327670 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -151,10 +151,11 @@ def filter_packages_by_interpreter(cls, packages, interpreter, platform): return [package for package in packages if package.compatible(interpreter.identity, platform)] - def __init__(self, allow_prereleases=None, interpreter=None, platform=None): + def __init__(self, allow_prereleases=None, interpreter=None, platform=None, pkg_blacklist=None): self._interpreter = interpreter or PythonInterpreter.get() self._platform = platform or Platform.current() self._allow_prereleases = allow_prereleases + self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {} def package_iterator(self, resolvable, existing=None): if existing: @@ -180,6 +181,12 @@ def build(self, package, options): 'Could not get distribution for %s on platform %s.' % (package, self._platform)) return dist + def _resolvable_is_blacklisted(self, resolvable_name): + return ( + resolvable_name in self._blacklist and + self._interpreter.identity.matches(self._blacklist[resolvable_name]) + ) + def resolve(self, resolvables, resolvable_set=None): resolvables = [(resolvable, None) for resolvable in resolvables] resolvable_set = resolvable_set or _ResolvableSet() @@ -193,7 +200,11 @@ def resolve(self, resolvables, resolvable_set=None): if resolvable in processed_resolvables: continue packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name)) - resolvable_set.merge(resolvable, packages, parent) + + # TODO: Remove blacklist strategy in favor of smart requirement handling + # https://github.com/pantsbuild/pex/issues/456 + if not self._resolvable_is_blacklisted(resolvable.name): + resolvable_set.merge(resolvable, packages, parent) processed_resolvables.add(resolvable) built_packages = {} @@ -299,7 +310,8 @@ def resolve(requirements, precedence=None, cache=None, cache_ttl=None, - allow_prereleases=None): + allow_prereleases=None, + pkg_blacklist=None): """Produce all distributions needed to (recursively) meet `requirements` :param requirements: An iterator of Requirement-like things, either @@ -329,6 +341,14 @@ def resolve(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. + :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to + an interpreter constraint. If a package name is in the blacklist and its interpreter + constraint matches the target interpreter, skip the requirement. This is needed to ensure + that universal requirement resolves for a target interpreter version do not error out on + interpreter specific requirements such as backport libs like `functools32`. + For example, a valid blacklist is {'functools32': 'CPython>3'}. + NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution + tracked by: https://github.com/pantsbuild/pex/issues/456 :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for @@ -367,11 +387,13 @@ def resolve(requirements, cache_ttl, allow_prereleases=allow_prereleases, interpreter=interpreter, - platform=platform) + platform=platform, + pkg_blacklist=pkg_blacklist) else: resolver = Resolver(allow_prereleases=allow_prereleases, interpreter=interpreter, - platform=platform) + platform=platform, + pkg_blacklist=pkg_blacklist) return resolver.resolve(resolvables_from_iterable(requirements, builder)) @@ -384,7 +406,8 @@ def resolve_multi(requirements, precedence=None, cache=None, cache_ttl=None, - allow_prereleases=None): + allow_prereleases=None, + pkg_blacklist=None): """A generator function that produces all distributions needed to meet `requirements` for multiple interpreters and/or platforms. @@ -415,6 +438,14 @@ def resolve_multi(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. + :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to + an interpreter constraint. If a package name is in the blacklist and its interpreter + constraint matches the target interpreter, skip the requirement. This is needed to ensure + that universal requirement resolves for a target interpreter version do not error out on + interpreter specific requirements such as backport libs like `functools32`. + For example, a valid blacklist is {'functools32': 'CPython>3'}. + NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution + tracked by: https://github.com/pantsbuild/pex/issues/456 :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for @@ -435,7 +466,8 @@ def resolve_multi(requirements, precedence, cache, cache_ttl, - allow_prereleases): + allow_prereleases, + pkg_blacklist=pkg_blacklist): if resolvable not in seen: seen.add(resolvable) yield resolvable diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 6ae290304..66d28efea 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -2,17 +2,20 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import sys import time import pytest from twitter.common.contextutil import temporary_dir from pex.common import safe_copy +from pex.compatibility import PY2 from pex.crawler import Crawler from pex.fetcher import Fetcher +from pex.interpreter import PythonInterpreter from pex.package import EggPackage, SourcePackage from pex.resolvable import ResolvableRequirement -from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi +from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve, resolve_multi from pex.resolver_options import ResolverOptionsBuilder from pex.testing import make_sdist @@ -349,3 +352,26 @@ def test_resolvable_set_built(): updated_rs.merge(rq, [binary_pkg]) assert updated_rs.get('foo') == set([binary_pkg]) assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)] + + +def test_resolver_blacklist(): + if PY2: + blacklist = {'project2': '<3'} + required_project = "project2;python_version>'3'" + else: + blacklist = {'project2': '>3'} + required_project = "project2;python_version<'3'" + + project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project]) + project2 = make_sdist(name='project2', version='1.1.0') + + with temporary_dir() as td: + safe_copy(project1, os.path.join(td, os.path.basename(project1))) + safe_copy(project2, os.path.join(td, os.path.basename(project2))) + fetchers = [Fetcher([td])] + + dists = resolve(['project1'], fetchers=fetchers) + assert len(dists) == 2 + + dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist) + assert len(dists) == 1