Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add blacklist handling for skipping requirements in pex resolver #457

Merged
merged 4 commits into from
Apr 12, 2018
Merged

Add blacklist handling for skipping requirements in pex resolver #457

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a1ffd82
Add blacklist handling for skipping requirements
Apr 5, 2018
3c95026
Address @UnrememberMe's comments
Apr 9, 2018
385e023
Address kwlzn comments
Apr 10, 2018
158ae60
Use required_project variable
Apr 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 35 additions & 7 deletions pex/resolver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,11 @@ def filter_packages_by_interpreter(cls, packages, interpreter, platform):
return [package for package in packages
if package.compatible(interpreter.identity, platform)]

def __init__(self, allow_prereleases=None, interpreter=None, platform=None):
def __init__(self, allow_prereleases=None, interpreter=None, platform=None, pkg_blacklist=None):
self._interpreter = interpreter or PythonInterpreter.get()
self._platform = platform or Platform.current()
self._allow_prereleases = allow_prereleases
self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {}

def package_iterator(self, resolvable, existing=None):
if existing:
Expand All @@ -180,6 +181,12 @@ def build(self, package, options):
'Could not get distribution for %s on platform %s.' % (package, self._platform))
return dist

def resolvable_is_blacklisted(self, resolvable_name):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should probably be private, especially since this solution is meant to be temporary.

return (
resolvable_name in self._blacklist and
self._interpreter.identity.matches(self._blacklist[resolvable_name])
)

def resolve(self, resolvables, resolvable_set=None):
resolvables = [(resolvable, None) for resolvable in resolvables]
resolvable_set = resolvable_set or _ResolvableSet()
Expand All @@ -193,7 +200,11 @@ def resolve(self, resolvables, resolvable_set=None):
if resolvable in processed_resolvables:
continue
packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name))
resolvable_set.merge(resolvable, packages, parent)

# TODO: Remove blacklist strategy in favor of smart requirement handling
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe

if not self.resolvable_is_blacklisted(resolvable.name):
  resolvable_set.merge(resolvable, packages, parent)
processed_resolvables.add(resolvable)

seems cleaner to me.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this TODO is good, but it'd be great to make the temporary nature of this change more evident in the docstring(s) below where developers are more likely to look. Would be good to include the ticket in there as well.

# https://github.com/pantsbuild/pex/issues/456
if not self.resolvable_is_blacklisted(resolvable.name):
resolvable_set.merge(resolvable, packages, parent)
processed_resolvables.add(resolvable)

built_packages = {}
Expand Down Expand Up @@ -299,7 +310,8 @@ def resolve(requirements,
precedence=None,
cache=None,
cache_ttl=None,
allow_prereleases=None):
allow_prereleases=None,
pkg_blacklist=None):
"""Produce all distributions needed to (recursively) meet `requirements`

:param requirements: An iterator of Requirement-like things, either
Expand Down Expand Up @@ -329,6 +341,12 @@ def resolve(requirements,
``context``.
:keyword allow_prereleases: (optional) Include pre-release and development versions. If
unspecified only stable versions will be resolved, unless explicitly included.
:keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

would love to have a better documentation on why a package needed to be blacklisted

an interpreter constraint. If a package name is in the blacklist and its interpreter
constraint matches the target interpreter, skip the requirement. This is needed to ensure
that universal requirement resolves for a target interpreter version do not error out on
interpreter specific requirements such as backport libs like `functools32`.
For example, a valid blacklist is {'functools32': 'CPython>3'}.
:returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``.
:raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
:raises Untranslateable: If no compatible distributions could be acquired for
Expand Down Expand Up @@ -367,11 +385,13 @@ def resolve(requirements,
cache_ttl,
allow_prereleases=allow_prereleases,
interpreter=interpreter,
platform=platform)
platform=platform,
pkg_blacklist=pkg_blacklist)
else:
resolver = Resolver(allow_prereleases=allow_prereleases,
interpreter=interpreter,
platform=platform)
platform=platform,
pkg_blacklist=pkg_blacklist)

return resolver.resolve(resolvables_from_iterable(requirements, builder))

Expand All @@ -384,7 +404,8 @@ def resolve_multi(requirements,
precedence=None,
cache=None,
cache_ttl=None,
allow_prereleases=None):
allow_prereleases=None,
pkg_blacklist=None):
"""A generator function that produces all distributions needed to meet `requirements`
for multiple interpreters and/or platforms.

Expand Down Expand Up @@ -415,6 +436,12 @@ def resolve_multi(requirements,
``context``.
:keyword allow_prereleases: (optional) Include pre-release and development versions. If
unspecified only stable versions will be resolved, unless explicitly included.
:keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to
an interpreter constraint. If a package name is in the blacklist and its interpreter
constraint matches the target interpreter, skip the requirement. This is needed to ensure
that universal requirement resolves for a target interpreter version do not error out on
interpreter specific requirements such as backport libs like `functools32`.
For example, a valid blacklist is {'functools32': 'CPython>3'}.
:yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``.
:raises Unsatisfiable: If ``requirements`` is not transitively satisfiable.
:raises Untranslateable: If no compatible distributions could be acquired for
Expand All @@ -435,7 +462,8 @@ def resolve_multi(requirements,
precedence,
cache,
cache_ttl,
allow_prereleases):
allow_prereleases,
pkg_blacklist=pkg_blacklist):
if resolvable not in seen:
seen.add(resolvable)
yield resolvable
34 changes: 33 additions & 1 deletion tests/test_resolver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,20 @@
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import os
import sys
import time

import pytest
from twitter.common.contextutil import temporary_dir

from pex.common import safe_copy
from pex.compatibility import PY2
from pex.crawler import Crawler
from pex.fetcher import Fetcher
from pex.interpreter import PythonInterpreter
from pex.package import EggPackage, SourcePackage
from pex.resolvable import ResolvableRequirement
from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi
from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve, resolve_multi
from pex.resolver_options import ResolverOptionsBuilder
from pex.testing import make_sdist

Expand Down Expand Up @@ -349,3 +352,32 @@ def test_resolvable_set_built():
updated_rs.merge(rq, [binary_pkg])
assert updated_rs.get('foo') == set([binary_pkg])
assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)]

def test_resolver_blacklist():
project = make_sdist(name='project', version='1.0.0')
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this probably needs a setup.py-sourced transitive dep to blacklist as a complete test case?

other_project = make_sdist(name='other_project', version='1.1.0')

with temporary_dir() as td:
safe_copy(project, os.path.join(td, os.path.basename(project)))
safe_copy(other_project, os.path.join(td, os.path.basename(other_project)))
fetchers = [Fetcher([td])]

if PY2:
blacklist = {'project': '<3'}
else:
blacklist = {'project': '>3'}

dists = resolve(['project'], fetchers=fetchers, pkg_blacklist=blacklist)
assert len(dists) == 0

dists = resolve(['other_project'], fetchers=fetchers, pkg_blacklist=blacklist)
assert len(dists) == 1
assert '1.1.0' == dists[0].version

dists = list(resolve_multi(['project'], fetchers=fetchers, pkg_blacklist=blacklist))
assert len(dists) == 0

dists = list(resolve_multi(['project', 'other_project'], fetchers=fetchers,
pkg_blacklist=blacklist))
assert len(dists) == 1
assert '1.1.0' == dists[0].version