From 65f7c5e34ea564c69d9a44c628b1868bcb575739 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 23 Aug 2021 16:18:36 -0700 Subject: [PATCH] Clean up resolver internal APIs. (#1420) Kill ~unused resolve and rename resolve_multi to resolve. Eliminate ResolvedDistribution alias in favor of InstalledDistribution which better reflects the shap of the distribution (not a wheel, but an installed wheel). Fix test-use fallout. Work towards #1401. Closes #969 --- pex/bin/pex.py | 4 +- pex/pip.py | 11 +- pex/resolver.py | 348 ++++++++++++-------------------------- tests/test_bdist_pex.py | 4 +- tests/test_environment.py | 31 ++-- tests/test_pex.py | 14 +- tests/test_resolver.py | 178 +++++++++---------- 7 files changed, 233 insertions(+), 357 deletions(-) diff --git a/pex/bin/pex.py b/pex/bin/pex.py index b965d9043..ebdff4dc1 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -31,7 +31,7 @@ from pex.pex_builder import CopyMode, PEXBuilder from pex.pip import ResolverVersion from pex.platforms import Platform -from pex.resolver import Unsatisfiable, parsed_platform, resolve_from_pex, resolve_multi +from pex.resolver import Unsatisfiable, parsed_platform, resolve, resolve_from_pex from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.variables import ENV, Variables @@ -1024,7 +1024,7 @@ def to_python_interpreter(full_path_or_basename): ) else: with TRACER.timed("Resolving requirements."): - resolveds = resolve_multi( + resolveds = resolve( requirements=reqs, requirement_files=options.requirement_files, constraint_files=options.constraint_files, diff --git a/pex/pip.py b/pex/pip.py index 54c353cd0..cf3f7231f 100644 --- a/pex/pip.py +++ b/pex/pip.py @@ -20,7 +20,7 @@ from textwrap import dedent from pex import dist_metadata, third_party -from pex.common import atomic_directory, is_script, safe_mkdtemp +from pex.common import atomic_directory, safe_mkdtemp from pex.compatibility import urlparse from pex.dist_metadata import ProjectNameAndVersion from pex.distribution_target import DistributionTarget @@ -51,6 +51,7 @@ Mapping, Optional, Protocol, + Sequence, Tuple, Union, ) @@ -96,8 +97,8 @@ def for_value(cls, value): class PackageIndexConfiguration(object): @staticmethod def _calculate_args( - indexes=None, # type: Optional[List[str]] - find_links=None, # type: Optional[List[str]] + indexes=None, # type: Optional[Sequence[str]] + find_links=None, # type: Optional[Iterable[str]] network_configuration=None, # type: Optional[NetworkConfiguration] ): # type: (...) -> Iterator[str] @@ -171,8 +172,8 @@ def _calculate_env( def create( cls, resolver_version=None, # type: Optional[ResolverVersion.Value] - indexes=None, # type: Optional[List[str]] - find_links=None, # type: Optional[List[str]] + indexes=None, # type: Optional[Sequence[str]] + find_links=None, # type: Optional[Iterable[str]] network_configuration=None, # type: Optional[NetworkConfiguration] ): # type: (...) -> PackageIndexConfiguration diff --git a/pex/resolver.py b/pex/resolver.py index f82004de9..43af01db7 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -8,7 +8,7 @@ import itertools import os import zipfile -from collections import OrderedDict, defaultdict +from collections import OrderedDict, Sequence, defaultdict from pex.common import AtomicDirectory, atomic_directory, safe_mkdtemp from pex.distribution_target import DistributionTarget @@ -20,7 +20,7 @@ from pex.orderedset import OrderedSet from pex.pep_503 import ProjectName, distribution_satisfies_requirement from pex.pex_info import PexInfo -from pex.pip import PackageIndexConfiguration, get_pip +from pex.pip import PackageIndexConfiguration, ResolverVersion, get_pip from pex.platforms import Platform from pex.requirements import ( Constraint, @@ -30,7 +30,7 @@ ) from pex.third_party.pkg_resources import Distribution, Requirement from pex.tracer import TRACER -from pex.typing import TYPE_CHECKING, cast +from pex.typing import TYPE_CHECKING from pex.util import CacheHelper, DistributionHelper if TYPE_CHECKING: @@ -71,10 +71,6 @@ def with_direct_requirement(self, direct_requirement=None): ) -# A type alias to preserve API compatibility for resolve and resolve_multi. -ResolvedDistribution = InstalledDistribution - - def parsed_platform(platform=None): # type: (Optional[Union[str, Platform]]) -> Optional[Platform] """Parse the given platform into a `Platform` object. @@ -89,33 +85,24 @@ def parsed_platform(platform=None): return Platform.create(platform) if platform and platform != "current" else None +def _uniqued_targets(targets=None): + # type: (Optional[Iterable[DistributionTarget]]) -> Tuple[DistributionTarget, ...] + return tuple(OrderedSet(targets)) if targets is not None else () + + +@attr.s(frozen=True) class DownloadRequest(object): - def __init__( - self, - targets, # type: OrderedSet[DistributionTarget] - direct_requirements, # type: Iterable[ParsedRequirement] - requirements=None, # type: Optional[Iterable[str]] - requirement_files=None, # type: Optional[Iterable[str]] - constraint_files=None, # type: Optional[Iterable[str]] - allow_prereleases=False, # type: bool - transitive=True, # type: bool - package_index_configuration=None, # type: Optional[PackageIndexConfiguration] - cache=None, # type: Optional[str] - build=True, # type: bool - use_wheel=True, # type: bool - ): - # type: (...) -> None - self.targets = tuple(targets) - self.direct_requirements = direct_requirements - self.requirements = requirements - self.requirement_files = requirement_files - self.constraint_files = constraint_files - self.allow_prereleases = allow_prereleases - self.transitive = transitive - self.package_index_configuration = package_index_configuration - self.cache = cache - self.build = build - self.use_wheel = use_wheel + targets = attr.ib(converter=_uniqued_targets) # type: Tuple[DistributionTarget, ...] + direct_requirements = attr.ib() # type: Iterable[ParsedRequirement] + requirements = attr.ib(default=None) # type: Optional[Iterable[str]] + requirement_files = attr.ib(default=None) # type: Optional[Iterable[str]] + constraint_files = attr.ib(default=None) # type: Optional[Iterable[str]] + allow_prereleases = attr.ib(default=False) # type: bool + transitive = attr.ib(default=True) # type: bool + package_index_configuration = attr.ib(default=None) # type: Optional[PackageIndexConfiguration] + cache = attr.ib(default=None) # type: Optional[str] + build = attr.ib(default=True) # type: bool + use_wheel = attr.ib(default=True) # type: bool def iter_local_projects(self): # type: () -> Iterator[BuildRequest] @@ -165,20 +152,15 @@ def _spawn_download( return SpawnedJob.wait(job=download_job, result=DownloadResult(target, download_dir)) +@attr.s(frozen=True) class DownloadResult(object): @staticmethod def _is_wheel(path): # type: (str) -> bool return os.path.isfile(path) and path.endswith(".whl") - def __init__( - self, - target, # type: DistributionTarget - download_dir, # type: str - ): - # type: (...) -> None - self.target = target - self.download_dir = download_dir + target = attr.ib() # type: DistributionTarget + download_dir = attr.ib() # type: str def _iter_distribution_paths(self): # type: () -> Iterator[str] @@ -740,7 +722,7 @@ def _check_install(self, installed_distributions): installed_distribution_by_project_name = OrderedDict( (ProjectName(resolved_distribution.distribution), resolved_distribution) for resolved_distribution in installed_distributions - ) # type: OrderedDict[ProjectName, ResolvedDistribution] + ) # type: OrderedDict[ProjectName, InstalledDistribution] unsatisfied = [] for installed_distribution in installed_distribution_by_project_name.values(): @@ -781,100 +763,6 @@ def _check_install(self, installed_distributions): ) -def resolve( - requirements=None, - requirement_files=None, - constraint_files=None, - allow_prereleases=False, - transitive=True, - interpreter=None, - platform=None, - indexes=None, - find_links=None, - network_configuration=None, - cache=None, - build=True, - use_wheel=True, - compile=False, - manylinux=None, - max_parallel_jobs=None, - ignore_errors=False, -): - """Produce all distributions needed to meet all specified requirements. - - :keyword requirements: A sequence of requirement strings. - :type requirements: list of str - :keyword requirement_files: A sequence of requirement file paths. - :type requirement_files: list of str - :keyword constraint_files: A sequence of constraint file paths. - :type constraint_files: list of str - :keyword bool allow_prereleases: Whether to include pre-release and development versions when - resolving requirements. Defaults to ``False``, but any requirements that explicitly request - prerelease or development versions will override this setting. - :keyword bool transitive: Whether to resolve transitive dependencies of requirements. - Defaults to ``True``. - :keyword interpreter: If specified, distributions will be resolved for this interpreter, and - non-wheel distributions will be built against this interpreter. If both `interpreter` and - `platform` are ``None`` (the default), this defaults to the current interpreter. - :type interpreter: :class:`pex.interpreter.PythonInterpreter` - :keyword str platform: The exact PEP425-compatible platform string to resolve distributions for, - in addition to the platform of the given interpreter, if provided. If any distributions need - to be built, use the interpreter argument instead, providing the corresponding interpreter. - However, if the platform matches the current interpreter, the current interpreter will be used - to build any non-wheels. - :keyword indexes: A list of urls or paths pointing to PEP 503 compliant repositories to search for - distributions. Defaults to ``None`` which indicates to use the default pypi index. To turn off - use of all indexes, pass an empty list. - :type indexes: list of str - :keyword find_links: A list or URLs, paths to local html files or directory paths. If URLs or - local html file paths, these are parsed for links to distributions. If a local directory path, - its listing is used to discover distributions. - :type find_links: list of str - :keyword network_configuration: Configuration for network requests made downloading and building - distributions. - :type network_configuration: :class:`pex.network_configuration.NetworkConfiguration` - :keyword str cache: A directory path to use to cache distributions locally. - :keyword bool build: Whether to allow building source distributions when no wheel is found. - Defaults to ``True``. - :keyword bool use_wheel: Whether to allow resolution of pre-built wheel distributions. - Defaults to ``True``. - :keyword bool compile: Whether to pre-compile resolved distribution python sources. - Defaults to ``False``. - :keyword str manylinux: The upper bound manylinux standard to support when targeting foreign linux - platforms. Defaults to ``None``. - :keyword int max_parallel_jobs: The maximum number of parallel jobs to use when resolving, - building and installing distributions in a resolve. Defaults to the number of CPUs available. - :keyword bool ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. - :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. - :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. - :raises Untranslatable: If no compatible distributions could be acquired for - a particular requirement. - :raises ValueError: If a foreign `platform` was provided, and `use_wheel=False`. - :raises ValueError: If `build=False` and `use_wheel=False`. - """ - # TODO(https://github.com/pantsbuild/pex/issues/969): Deprecate resolve with a single interpreter - # or platform and rename resolve_multi to resolve for a single API entrypoint to a full resolve. - return resolve_multi( - requirements=requirements, - requirement_files=requirement_files, - constraint_files=constraint_files, - allow_prereleases=allow_prereleases, - transitive=transitive, - interpreters=None if interpreter is None else [interpreter], - platforms=None if platform is None else [platform], - indexes=indexes, - find_links=find_links, - network_configuration=network_configuration, - cache=cache, - build=build, - use_wheel=use_wheel, - compile=compile, - manylinux=manylinux, - max_parallel_jobs=max_parallel_jobs, - ignore_errors=ignore_errors, - ) - - def _parse_reqs( requirements=None, # type: Optional[Iterable[str]] requirement_files=None, # type: Optional[Iterable[str]] @@ -897,81 +785,73 @@ def _parse_reqs( return parsed_requirements -def resolve_multi( - requirements=None, - requirement_files=None, - constraint_files=None, - allow_prereleases=False, - transitive=True, - interpreters=None, - platforms=None, - indexes=None, - find_links=None, - resolver_version=None, - network_configuration=None, - cache=None, - build=True, - use_wheel=True, - compile=False, - manylinux=None, - max_parallel_jobs=None, - ignore_errors=False, - verify_wheels=True, +def resolve( + requirements=None, # type: Optional[Iterable[str]] + requirement_files=None, # type: Optional[Iterable[str]] + constraint_files=None, # type: Optional[Iterable[str]] + allow_prereleases=False, # type: bool + transitive=True, # type: bool + interpreters=None, # type: Optional[Iterable[PythonInterpreter]] + platforms=None, # type: Optional[Iterable[Union[str, Platform]]] + indexes=None, # type: Optional[Sequence[str]] + find_links=None, # type: Optional[Iterable[str]] + resolver_version=None, # type: Optional[ResolverVersion.Value] + network_configuration=None, # type: Optional[NetworkConfiguration] + cache=None, # type: Optional[str] + build=True, # type: bool + use_wheel=True, # type: bool + compile=False, # type: bool + manylinux=None, # type: Optional[str] + max_parallel_jobs=None, # type: Optional[int] + ignore_errors=False, # type: bool + verify_wheels=True, # type: bool ): + # type: (...) -> List[InstalledDistribution] """Resolves all distributions needed to meet requirements for multiple distribution targets. The resulting distributions are installed in individual chroots that can be independently added to `sys.path` :keyword requirements: A sequence of requirement strings. - :type requirements: list of str :keyword requirement_files: A sequence of requirement file paths. - :type requirement_files: list of str :keyword constraint_files: A sequence of constraint file paths. - :type constraint_files: list of str - :keyword bool allow_prereleases: Whether to include pre-release and development versions when + :keyword allow_prereleases: Whether to include pre-release and development versions when resolving requirements. Defaults to ``False``, but any requirements that explicitly request prerelease or development versions will override this setting. - :keyword bool transitive: Whether to resolve transitive dependencies of requirements. + :keyword transitive: Whether to resolve transitive dependencies of requirements. Defaults to ``True``. :keyword interpreters: If specified, distributions will be resolved for these interpreters, and non-wheel distributions will be built against each interpreter. If both `interpreters` and `platforms` are ``None`` (the default) or an empty iterable, this defaults to a list containing only the current interpreter. - :type interpreters: list of :class:`pex.interpreter.PythonInterpreter` :keyword platforms: An iterable of PEP425-compatible platform strings to resolve distributions for, in addition to the platforms of any given interpreters. If any distributions need to be built, use the interpreters argument instead, providing the corresponding interpreter. However, if any platform matches the current interpreter, the current interpreter will be used to build any non-wheels for that platform. - :type platforms: list of str :keyword indexes: A list of urls or paths pointing to PEP 503 compliant repositories to search for distributions. Defaults to ``None`` which indicates to use the default pypi index. To turn off use of all indexes, pass an empty list. - :type indexes: list of str :keyword find_links: A list or URLs, paths to local html files or directory paths. If URLs or local html file paths, these are parsed for links to distributions. If a local directory path, its listing is used to discover distributions. - :type find_links: list of str :keyword resolver_version: The resolver version to use. - :type resolver_version: :class:`ResolverVersion` :keyword network_configuration: Configuration for network requests made downloading and building distributions. - :type network_configuration: :class:`pex.network_configuration.NetworkConfiguration` - :keyword str cache: A directory path to use to cache distributions locally. - :keyword bool build: Whether to allow building source distributions when no wheel is found. + :keyword cache: A directory path to use to cache distributions locally. + :keyword build: Whether to allow building source distributions when no wheel is found. Defaults to ``True``. - :keyword bool use_wheel: Whether to allow resolution of pre-built wheel distributions. + :keyword use_wheel: Whether to allow resolution of pre-built wheel distributions. Defaults to ``True``. - :keyword bool compile: Whether to pre-compile resolved distribution python sources. + :keyword compile: Whether to pre-compile resolved distribution python sources. Defaults to ``False``. - :keyword str manylinux: The upper bound manylinux standard to support when targeting foreign linux + :keyword manylinux: The upper bound manylinux standard to support when targeting foreign linux platforms. Defaults to ``None``. - :keyword int max_parallel_jobs: The maximum number of parallel jobs to use when resolving, + :keyword max_parallel_jobs: The maximum number of parallel jobs to use when resolving, building and installing distributions in a resolve. Defaults to the number of CPUs available. - :keyword bool ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. - :keyword bool verify_wheels: Whether to verify wheels have valid metadata. Defaults to ``True``. - :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. + :keyword ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. + :keyword verify_wheels: Whether to verify wheels have valid metadata. Defaults to ``True``. + :returns: The resolved distributions meeting all requirements and constraints. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslatable: If no compatible distributions could be acquired for a particular requirement. @@ -1034,7 +914,7 @@ def resolve_multi( max_parallel_jobs=max_parallel_jobs, ) - install_requests = [] + install_requests = [] # type: List[InstallRequest] for download_result in download_results: build_requests.extend(download_result.build_requests()) install_requests.extend(download_result.install_requests()) @@ -1152,68 +1032,60 @@ def is_wheel(self): def download( - requirements=None, - requirement_files=None, - constraint_files=None, - allow_prereleases=False, - transitive=True, - interpreters=None, - platforms=None, - indexes=None, - find_links=None, - resolver_version=None, - network_configuration=None, - cache=None, - build=True, - use_wheel=True, - manylinux=None, - dest=None, - max_parallel_jobs=None, + requirements=None, # type: Optional[Iterable[str]] + requirement_files=None, # type: Optional[Iterable[str]] + constraint_files=None, # type: Optional[Iterable[str]] + allow_prereleases=False, # type: bool + transitive=True, # type: bool + interpreters=None, # type: Optional[Iterable[PythonInterpreter]] + platforms=None, # type: Optional[Iterable[Union[str, Platform]]] + indexes=None, # type: Optional[Sequence[str]] + find_links=None, # type: Optional[Iterable[str]] + resolver_version=None, # type: Optional[ResolverVersion.Value] + network_configuration=None, # type: Optional[NetworkConfiguration] + cache=None, # type: Optional[str] + build=True, # type: bool + use_wheel=True, # type: bool + manylinux=None, # type: Optional[str] + dest=None, # type: Optional[str] + max_parallel_jobs=None, # type: Optional[int] ): + # type: (...) -> List[LocalDistribution] """Downloads all distributions needed to meet requirements for multiple distribution targets. :keyword requirements: A sequence of requirement strings. - :type requirements: list of str :keyword requirement_files: A sequence of requirement file paths. - :type requirement_files: list of str :keyword constraint_files: A sequence of constraint file paths. - :type constraint_files: list of str - :keyword bool allow_prereleases: Whether to include pre-release and development versions when + :keyword allow_prereleases: Whether to include pre-release and development versions when resolving requirements. Defaults to ``False``, but any requirements that explicitly request prerelease or development versions will override this setting. - :keyword bool transitive: Whether to resolve transitive dependencies of requirements. + :keyword transitive: Whether to resolve transitive dependencies of requirements. Defaults to ``True``. :keyword interpreters: If specified, distributions will be resolved for these interpreters. If both `interpreters` and `platforms` are ``None`` (the default) or an empty iterable, this defaults to a list containing only the current interpreter. - :type interpreters: list of :class:`pex.interpreter.PythonInterpreter` :keyword platforms: An iterable of PEP425-compatible platform strings to resolve distributions for, in addition to the platforms of any given interpreters. - :type platforms: list of str - :keyword indexes: A list of urls or paths pointing to PEP 503 compliant repositories to search for - distributions. Defaults to ``None`` which indicates to use the default pypi index. To turn off - use of all indexes, pass an empty list. - :type indexes: list of str - :keyword find_links: A list or URLs, paths to local html files or directory paths. If URLs or + :keyword indexes: A list of urls or paths pointing to PEP 503 compliant repositories to search + for distributions. Defaults to ``None`` which indicates to use the default pypi index. To turn + off use of all indexes, pass an empty list. + :keyword find_links: A list of URLs, paths to local html files or directory paths. If URLs or local html file paths, these are parsed for links to distributions. If a local directory path, its listing is used to discover distributions. - :type find_links: list of str :keyword resolver_version: The resolver version to use. - :type resolver_version: :class:`ResolverVersion` :keyword network_configuration: Configuration for network requests made downloading and building distributions. - :type network_configuration: :class:`pex.network_configuration.NetworkConfiguration` - :keyword str cache: A directory path to use to cache distributions locally. - :keyword bool build: Whether to allow building source distributions when no wheel is found. + :keyword cache: A directory path to use to cache distributions locally. + :keyword build: Whether to allow building source distributions when no wheel is found. Defaults to ``True``. - :keyword bool use_wheel: Whether to allow resolution of pre-built wheel distributions. + :keyword use_wheel: Whether to allow resolution of pre-built wheel distributions. Defaults to ``True``. - :keyword str manylinux: The upper bound manylinux standard to support when targeting foreign linux + :keyword manylinux: The upper bound manylinux standard to support when targeting foreign linux platforms. Defaults to ``None``. - :keyword str dest: A directory path to download distributions to. - :keyword int max_parallel_jobs: The maximum number of parallel jobs to use when resolving, + :keyword dest: A directory path to download distributions to. + :keyword max_parallel_jobs: The maximum number of parallel jobs to use when resolving, building and installing distributions in a resolve. Defaults to the number of CPUs available. - :returns: List of :class:`LocalDistribution` instances meeting ``requirements``. + :returns: The local distributions meeting all requirements and constraints. :raises Unsatisfiable: If the resolution of download of distributions fails for any reason. :raises ValueError: If a foreign platform was provided in `platforms`, and `use_wheel=False`. :raises ValueError: If `build=False` and `use_wheel=False`. @@ -1272,42 +1144,38 @@ def add_build_requests(requests): def install( - local_distributions, - indexes=None, - find_links=None, - resolver_version=None, - network_configuration=None, - cache=None, - compile=False, - max_parallel_jobs=None, - ignore_errors=False, - verify_wheels=True, + local_distributions, # type: Iterable[LocalDistribution] + indexes=None, # type: Optional[Sequence[str]] + find_links=None, # type: Optional[Iterable[str]] + resolver_version=None, # type: Optional[ResolverVersion.Value] + network_configuration=None, # type: Optional[NetworkConfiguration] + cache=None, # type: Optional[str] + compile=False, # type: bool + max_parallel_jobs=None, # type: Optional[int] + ignore_errors=False, # type: bool + verify_wheels=True, # type: bool ): + # type: (...) -> List[InstalledDistribution] """Installs distributions in individual chroots that can be independently added to `sys.path`. :keyword local_distributions: The local distributions to install. - :type local_distributions: list of :class:`LocalDistribution` :keyword indexes: A list of urls or paths pointing to PEP 503 compliant repositories to search for distributions. Defaults to ``None`` which indicates to use the default pypi index. To turn off use of all indexes, pass an empty list. - :type indexes: list of str :keyword find_links: A list or URLs, paths to local html files or directory paths. If URLs or local html file paths, these are parsed for links to distributions. If a local directory path, its listing is used to discover distributions. - :type find_links: list of str :keyword resolver_version: The resolver version to use. - :type resolver_version: :class:`ResolverVersion` :keyword network_configuration: Configuration for network requests made downloading and building distributions. - :type network_configuration: :class:`pex.network_configuration.NetworkConfiguration` - :keyword str cache: A directory path to use to cache distributions locally. - :keyword bool compile: Whether to pre-compile resolved distribution python sources. + :keyword cache: A directory path to use to cache distributions locally. + :keyword compile: Whether to pre-compile resolved distribution python sources. Defaults to ``False``. - :keyword int max_parallel_jobs: The maximum number of parallel jobs to use when resolving, + :keyword max_parallel_jobs: The maximum number of parallel jobs to use when resolving, building and installing distributions in a resolve. Defaults to the number of CPUs available. - :keyword bool ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. - :keyword bool verify_wheels: Whether to verify wheels have valid metadata. Defaults to ``True``. - :returns: List of :class:`InstalledDistribution` instances meeting ``requirements``. + :keyword ignore_errors: Whether to ignore resolution solver errors. Defaults to ``False``. + :keyword verify_wheels: Whether to verify wheels have valid metadata. Defaults to ``True``. + :returns: The installed distributions meeting all requirements and constraints. :raises Untranslatable: If no compatible distributions could be acquired for a particular requirement. :raises Unsatisfiable: If not ignoring errors and distribution requirements are found to not be @@ -1356,7 +1224,7 @@ def resolve_from_pex( manylinux=None, # type: Optional[str] ignore_errors=False, # type: bool ): - # type: (...) -> List[ResolvedDistribution] + # type: (...) -> List[InstalledDistribution] direct_requirements = _parse_reqs(requirements, requirement_files, network_configuration) direct_requirements_by_project_name = ( @@ -1393,7 +1261,7 @@ def resolve_from_pex( unique_targets = _unique_targets( interpreters=interpreters, platforms=platforms, manylinux=manylinux ) - resolved_distributions = OrderedSet() # type: OrderedSet[ResolvedDistribution] + resolved_distributions = OrderedSet() # type: OrderedSet[InstalledDistribution] for target in unique_targets: pex_env = PEXEnvironment(pex, target=target) try: @@ -1423,6 +1291,6 @@ def resolve_from_pex( ) resolved_distributions.add( - ResolvedDistribution(target, distribution, direct_requirement=direct_requirement) + InstalledDistribution(target, distribution, direct_requirement=direct_requirement) ) return list(resolved_distributions) diff --git a/tests/test_bdist_pex.py b/tests/test_bdist_pex.py index 2a958c4f5..9a843f41f 100644 --- a/tests/test_bdist_pex.py +++ b/tests/test_bdist_pex.py @@ -41,8 +41,8 @@ def bdist_pex_pythonpath(): # Although the setuptools version is not important, we pick one so the test can leverage the # pex cache for speed run over run. BDIST_PEX_PYTHONPATH.extend( - resolved_distribution.distribution.location - for resolved_distribution in resolver.resolve(["setuptools==36.2.7"]) + installed_distribution.distribution.location + for installed_distribution in resolver.resolve(["setuptools==36.2.7"]) ) return BDIST_PEX_PYTHONPATH diff --git a/tests/test_environment.py b/tests/test_environment.py index bf002e602..3c491461e 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -19,7 +19,6 @@ from pex.pex import PEX from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo -from pex.resolver import resolve from pex.testing import ( IS_LINUX, IS_PYPY3, @@ -155,10 +154,12 @@ def main(): def add_requirements(builder, cache): # type: (PEXBuilder, str) -> None - for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter): - builder.add_distribution(resolved_dist.distribution) - if resolved_dist.direct_requirement: - builder.add_requirement(resolved_dist.direct_requirement) + for installed_dist in resolver.resolve( + requirements, cache=cache, interpreters=[builder.interpreter] + ): + builder.add_distribution(installed_dist.distribution) + if installed_dist.direct_requirement: + builder.add_requirement(installed_dist.direct_requirement) def add_wheel(builder, content): # type: (PEXBuilder, Dict[str, str]) -> None @@ -321,10 +322,10 @@ def bad_interpreter(): with yield_pex_builder( interpreter=bad_interpreter() ) as pb, temporary_filename() as pex_file: - for resolved_dist in resolver.resolve( - ["psutil==5.4.3"], cache=cache, interpreter=pb.interpreter + for installed_dist in resolver.resolve( + ["psutil==5.4.3"], cache=cache, interpreters=[pb.interpreter] ): - pb.add_dist_location(resolved_dist.distribution.location) + pb.add_dist_location(installed_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. @@ -377,10 +378,12 @@ def run(args, **env): def test_activate_extras_issue_615(): # type: () -> None with yield_pex_builder() as pb: - for resolved_dist in resolver.resolve(["pex[requests]==1.6.3"], interpreter=pb.interpreter): - if resolved_dist.direct_requirement: - pb.add_requirement(resolved_dist.direct_requirement) - pb.add_dist_location(resolved_dist.distribution.location) + for installed_dist in resolver.resolve( + ["pex[requests]==1.6.3"], interpreters=[pb.interpreter] + ): + if installed_dist.direct_requirement: + pb.add_requirement(installed_dist.direct_requirement) + pb.add_dist_location(installed_dist.distribution.location) pb.set_script("pex") pb.freeze() process = PEX(pb.path(), interpreter=pb.interpreter).run( @@ -401,8 +404,8 @@ def assert_namespace_packages_warning(distribution, version, expected_warning): # type: (str, str, bool) -> None requirement = "{}=={}".format(distribution, version) pb = PEXBuilder() - for resolved_dist in resolver.resolve([requirement]): - pb.add_dist_location(resolved_dist.distribution.location) + for installed_dist in resolver.resolve([requirement]): + pb.add_dist_location(installed_dist.distribution.location) pb.freeze() process = PEX(pb.path()).run(args=["-c", ""], blocking=False, stderr=subprocess.PIPE) diff --git a/tests/test_pex.py b/tests/test_pex.py index 4f913646a..0dc103655 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -13,13 +13,13 @@ import pytest +from pex import resolver from pex.common import safe_mkdir, safe_open, temporary_dir from pex.compatibility import PY2, WINDOWS, to_bytes from pex.interpreter import PythonInterpreter from pex.pex import PEX from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo -from pex.resolver import resolve from pex.testing import ( IS_PYPY3, PY27, @@ -714,8 +714,8 @@ def test_pex_run_strip_env(): def test_pex_run_custom_setuptools_useable(): # type: () -> None - resolved_dists = resolve(["setuptools==36.2.7"]) - dists = [resolved_dist.distribution for resolved_dist in resolved_dists] + installed_dists = resolver.resolve(["setuptools==36.2.7"]) + dists = [installed_dist.distribution for installed_dist in installed_dists] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, @@ -737,8 +737,8 @@ def test_pex_run_conflicting_custom_setuptools_useable(): # > pkg_resources/py31compat.py # > pkg_resources/_vendor/appdirs.py - resolved_dists = resolve(["setuptools==20.3.1"]) - dists = [resolved_dist.distribution for resolved_dist in resolved_dists] + installed_dists = resolver.resolve(["setuptools==20.3.1"]) + dists = [installed_dist.distribution for installed_dist in installed_dists] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, @@ -769,8 +769,8 @@ def test_pex_run_conflicting_custom_setuptools_useable(): def test_pex_run_custom_pex_useable(): # type: () -> None old_pex_version = "0.7.0" - resolved_dists = resolve(["pex=={}".format(old_pex_version), "setuptools==40.6.3"]) - dists = [resolved_dist.distribution for resolved_dist in resolved_dists] + installed_dists = resolver.resolve(["pex=={}".format(old_pex_version), "setuptools==40.6.3"]) + dists = [installed_dist.distribution for installed_dist in installed_dists] with temporary_dir() as temp_dir: from pex.version import __version__ diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 8a6897871..6dd134d48 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -24,8 +24,8 @@ Unsatisfiable, download, install, + resolve, resolve_from_pex, - resolve_multi, ) from pex.testing import ( IS_LINUX, @@ -71,21 +71,21 @@ def build_wheel(**kwargs): return whl -def local_resolve_multi(*args, **kwargs): +def local_resolve(*args, **kwargs): # type: (*Any, **Any) -> List[InstalledDistribution] # Skip remote lookups. kwargs["indexes"] = [] - return list(resolve_multi(*args, **kwargs)) + return list(resolve(*args, **kwargs)) def test_empty_resolve(): # type: () -> None - empty_resolve_multi = local_resolve_multi([]) - assert empty_resolve_multi == [] + empty_resolve = local_resolve([]) + assert empty_resolve == [] with temporary_dir() as td: - empty_resolve_multi = local_resolve_multi([], cache=td) - assert empty_resolve_multi == [] + empty_resolve = local_resolve([], cache=td) + assert empty_resolve == [] def test_simple_local_resolve(): @@ -94,8 +94,8 @@ def test_simple_local_resolve(): with temporary_dir() as td: safe_copy(project_wheel, os.path.join(td, os.path.basename(project_wheel))) - resolved_dists = local_resolve_multi(["project"], find_links=[td]) - assert len(resolved_dists) == 1 + installed_dists = local_resolve(["project"], find_links=[td]) + assert len(installed_dists) == 1 def test_resolve_cache(): @@ -106,20 +106,22 @@ def test_resolve_cache(): safe_copy(project_wheel, os.path.join(td, os.path.basename(project_wheel))) # Without a cache, each resolve should be isolated, but otherwise identical. - resolved_dists1 = local_resolve_multi(["project"], find_links=[td]) - resolved_dists2 = local_resolve_multi(["project"], find_links=[td]) - assert resolved_dists1 != resolved_dists2 - assert len(resolved_dists1) == 1 - assert len(resolved_dists2) == 1 - assert resolved_dists1[0].direct_requirement == resolved_dists2[0].direct_requirement - assert resolved_dists1[0].distribution.location != resolved_dists2[0].distribution.location + installed_dists1 = local_resolve(["project"], find_links=[td]) + installed_dists2 = local_resolve(["project"], find_links=[td]) + assert installed_dists1 != installed_dists2 + assert len(installed_dists1) == 1 + assert len(installed_dists2) == 1 + assert installed_dists1[0].direct_requirement == installed_dists2[0].direct_requirement + assert ( + installed_dists1[0].distribution.location != installed_dists2[0].distribution.location + ) # With a cache, each resolve should be identical. - resolved_dists3 = local_resolve_multi(["project"], find_links=[td], cache=cache) - resolved_dists4 = local_resolve_multi(["project"], find_links=[td], cache=cache) - assert resolved_dists1 != resolved_dists3 - assert resolved_dists2 != resolved_dists3 - assert resolved_dists3 == resolved_dists4 + installed_dists3 = local_resolve(["project"], find_links=[td], cache=cache) + installed_dists4 = local_resolve(["project"], find_links=[td], cache=cache) + assert installed_dists1 != installed_dists3 + assert installed_dists2 != installed_dists3 + assert installed_dists3 == installed_dists4 def test_diamond_local_resolve_cached(): @@ -132,10 +134,8 @@ def test_diamond_local_resolve_cached(): for wheel in (project1_wheel, project2_wheel): safe_copy(wheel, os.path.join(dd, os.path.basename(wheel))) with temporary_dir() as cd: - resolved_dists = local_resolve_multi( - ["project1", "project2"], find_links=[dd], cache=cd - ) - assert len(resolved_dists) == 2 + installed_dists = local_resolve(["project1", "project2"], find_links=[dd], cache=cd) + assert len(installed_dists) == 2 def test_cached_dependency_pinned_unpinned_resolution_multi_run(): @@ -149,15 +149,15 @@ def test_cached_dependency_pinned_unpinned_resolution_multi_run(): safe_copy(wheel, os.path.join(td, os.path.basename(wheel))) with temporary_dir() as cd: # First run, pinning 1.0.0 in the cache - resolved_dists = local_resolve_multi(["project==1.0.0"], find_links=[td], cache=cd) - assert len(resolved_dists) == 1 - assert resolved_dists[0].distribution.version == "1.0.0" + installed_dists = local_resolve(["project==1.0.0"], find_links=[td], cache=cd) + assert len(installed_dists) == 1 + assert installed_dists[0].distribution.version == "1.0.0" # Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also # return SourcePackages found in td - resolved_dists = local_resolve_multi(["project"], find_links=[td], cache=cd) - assert len(resolved_dists) == 1 - assert resolved_dists[0].distribution.version == "1.1.0" + installed_dists = local_resolve(["project"], find_links=[td], cache=cd) + assert len(installed_dists) == 1 + assert installed_dists[0].distribution.version == "1.1.0" def test_intransitive(): @@ -169,10 +169,10 @@ def test_intransitive(): for wheel in (foo1_0, bar1_0): safe_copy(wheel, os.path.join(td, os.path.basename(wheel))) with temporary_dir() as cd: - resolved_dists = local_resolve_multi( + installed_dists = local_resolve( ["foo", "bar"], find_links=[td], cache=cd, transitive=False ) - assert len(resolved_dists) == 2 + assert len(installed_dists) == 2 def test_resolve_prereleases(): @@ -185,10 +185,10 @@ def test_resolve_prereleases(): safe_copy(wheel, os.path.join(td, os.path.basename(wheel))) def assert_resolve(expected_version, **resolve_kwargs): - resolved_dists = local_resolve_multi(["dep>=1,<4"], find_links=[td], **resolve_kwargs) - assert 1 == len(resolved_dists) - resolved_dist = resolved_dists[0] - assert expected_version == resolved_dist.distribution.version + installed_dists = local_resolve(["dep>=1,<4"], find_links=[td], **resolve_kwargs) + assert 1 == len(installed_dists) + installed_dist = installed_dists[0] + assert expected_version == installed_dist.distribution.version assert_resolve("2.0.0") assert_resolve("2.0.0", allow_prereleases=False) @@ -211,10 +211,10 @@ def test_resolve_extra_setup_py(): with temporary_dir() as td: safe_copy(project2_wheel, os.path.join(td, os.path.basename(project2_wheel))) - resolved_dists = local_resolve_multi(["{}[foo]".format(project1_dir)], find_links=[td]) + installed_dists = local_resolve(["{}[foo]".format(project1_dir)], find_links=[td]) assert {_parse_requirement(req) for req in ("project1==1.0.0", "project2==2.0.0")} == { - _parse_requirement(resolved_dist.distribution.as_requirement()) - for resolved_dist in resolved_dists + _parse_requirement(installed_dist.distribution.as_requirement()) + for installed_dist in installed_dists } @@ -228,18 +228,18 @@ def test_resolve_extra_wheel(): for wheel in (project1_wheel, project2_wheel): safe_copy(wheel, os.path.join(td, os.path.basename(wheel))) - resolved_dists = local_resolve_multi(["project1[foo]"], find_links=[td]) + installed_dists = local_resolve(["project1[foo]"], find_links=[td]) assert {_parse_requirement(req) for req in ("project1==1.0.0", "project2==2.0.0")} == { - _parse_requirement(resolved_dist.distribution.as_requirement()) - for resolved_dist in resolved_dists + _parse_requirement(installed_dist.distribution.as_requirement()) + for installed_dist in installed_dists } def resolve_wheel_names(**kwargs): # type: (**Any) -> List[str] return [ - os.path.basename(resolved_distribution.distribution.location) - for resolved_distribution in resolve_multi(**kwargs) + os.path.basename(installed_distribution.distribution.location) + for installed_distribution in resolve(**kwargs) ] @@ -353,10 +353,13 @@ def test_issues_851(): def resolve_pytest(python_version, pytest_version): interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(python_version)) - resolved_dists = resolve_multi( + installed_dists = resolve( interpreters=[interpreter], requirements=["pytest=={}".format(pytest_version)] ) - project_to_version = {rd.distribution.key: rd.distribution.version for rd in resolved_dists} + project_to_version = { + installed_dist.distribution.key: installed_dist.distribution.version + for installed_dist in installed_dists + } assert project_to_version["pytest"] == pytest_version return project_to_version @@ -393,7 +396,7 @@ def test_issues_892(): python27 = PythonInterpreter.from_binary({python27!r}) - result = resolver.resolve(requirements=['packaging==19.2'], interpreter=python27) + result = resolver.resolve(requirements=['packaging==19.2'], interpreters=[python27]) print('Resolved: {{}}'.format(result)) """.format( python27=python27 @@ -520,15 +523,15 @@ def test_resolve_arbitrary_equality_issues_940(): version="1.0.2-fba4511", python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*", ) - resolved_distributions = local_resolve_multi( + installed_distributions = local_resolve( requirements=[dist], # We need this to allow the invalid version above to sneak by pip wheel metadata # verification. verify_wheels=False, ) - assert len(resolved_distributions) == 1 - requirement = resolved_distributions[0].direct_requirement + assert len(installed_distributions) == 1 + requirement = installed_distributions[0].direct_requirement assert requirement is not None, ( "The foo requirement was direct; so the resulting resolved distribution should carry the " "associated requirement." @@ -539,22 +542,22 @@ def test_resolve_arbitrary_equality_issues_940(): def test_resolve_overlapping_requirements_discriminated_by_markers_issues_1196(py27): # type: (PythonInterpreter) -> None - resolved_distributions = resolve_multi( + installed_distributions = resolve( requirements=[ "setuptools<45; python_full_version == '2.7.*'", "setuptools; python_version > '2.7'", ], interpreters=[py27], ) - assert 1 == len(resolved_distributions) - resolved_distribution = resolved_distributions[0] + assert 1 == len(installed_distributions) + installed_distribution = installed_distributions[0] assert ( Requirement.parse("setuptools<45; python_full_version == '2.7.*'") - == resolved_distribution.direct_requirement + == installed_distribution.direct_requirement ) assert ( Requirement.parse("setuptools==44.1.1") - == resolved_distribution.distribution.as_requirement() + == installed_distribution.distribution.as_requirement() ) @@ -568,7 +571,7 @@ def create_pex_repository( ): # type: (...) -> str pex_builder = PEXBuilder() - for resolved_dist in resolve_multi( + for installed_dist in resolve( interpreters=interpreters, platforms=platforms, requirements=requirements, @@ -576,9 +579,9 @@ def create_pex_repository( constraint_files=constraint_files, manylinux=manylinux, ): - pex_builder.add_distribution(resolved_dist.distribution) - if resolved_dist.direct_requirement: - pex_builder.add_requirement(resolved_dist.direct_requirement) + pex_builder.add_distribution(installed_dist.distribution) + if installed_dist.direct_requirement: + pex_builder.add_requirement(installed_dist.direct_requirement) pex_builder.freeze() return os.path.realpath(cast(str, pex_builder.path())) @@ -660,7 +663,7 @@ def test_resolve_from_pex( direct_requirements = pex_info.requirements assert 1 == len(direct_requirements) - resolved_distributions = resolve_from_pex( + installed_distributions = resolve_from_pex( pex=pex_repository, requirements=direct_requirements, interpreters=[py27, py38], @@ -669,9 +672,9 @@ def test_resolve_from_pex( ) distribution_locations_by_key = defaultdict(set) # type: DefaultDict[str, Set[str]] - for resolved_distribution in resolved_distributions: - distribution_locations_by_key[resolved_distribution.distribution.key].add( - resolved_distribution.distribution.location + for installed_distribution in installed_distributions: + distribution_locations_by_key[installed_distribution.distribution.key].add( + installed_distribution.distribution.location ) assert { @@ -707,7 +710,7 @@ def test_resolve_from_pex_subset( ): # type: (...) -> None - resolved_distributions = resolve_from_pex( + installed_distributions = resolve_from_pex( pex=pex_repository, requirements=["cffi"], platforms=[foreign_platform], @@ -715,7 +718,8 @@ def test_resolve_from_pex_subset( ) assert {"cffi", "pycparser"} == { - resolved_distribution.distribution.key for resolved_distribution in resolved_distributions + installed_distribution.distribution.key + for installed_distribution in installed_distributions } @@ -760,7 +764,7 @@ def test_resolve_from_pex_intransitive( ): # type: (...) -> None - resolved_distributions = resolve_from_pex( + installed_distributions = resolve_from_pex( pex=pex_repository, requirements=["requests"], transitive=False, @@ -769,23 +773,23 @@ def test_resolve_from_pex_intransitive( manylinux=manylinux, ) assert 3 == len( - resolved_distributions + installed_distributions ), "Expected one resolved distribution per distribution target." assert 1 == len( frozenset( - resolved_distribution.distribution.location - for resolved_distribution in resolved_distributions + installed_distribution.distribution.location + for installed_distribution in installed_distributions ) ), ( "Expected one underlying resolved universal distribution usable on Linux and macOs by " "both Python 2.7 and Python 3.6." ) - for resolved_distribution in resolved_distributions: + for installed_distribution in installed_distributions: assert ( Requirement.parse("requests==2.25.1") - == resolved_distribution.distribution.as_requirement() + == installed_distribution.distribution.as_requirement() ) - assert Requirement.parse("requests") == resolved_distribution.direct_requirement + assert Requirement.parse("requests") == installed_distribution.direct_requirement def test_resolve_from_pex_constraints( @@ -814,26 +818,26 @@ def test_resolve_from_pex_ignore_errors( # type: (...) -> None # See test_resolve_from_pex_constraints above for the failure this would otherwise cause. - resolved_distributions = resolve_from_pex( + installed_distributions = resolve_from_pex( pex=pex_repository, requirements=["requests"], constraint_files=[create_constraints_file("urllib3==1.26.2")], interpreters=[py27], ignore_errors=True, ) - resolved_distributions_by_key = { - resolved_distribution.distribution.key: resolved_distribution.distribution.as_requirement() - for resolved_distribution in resolved_distributions + installed_distributions_by_key = { + installed_distribution.distribution.key: installed_distribution.distribution.as_requirement() + for installed_distribution in installed_distributions } - assert len(resolved_distributions_by_key) > 1, "We should resolve at least requests and urllib3" - assert "requests" in resolved_distributions_by_key - assert Requirement.parse("urllib3==1.26.1") == resolved_distributions_by_key["urllib3"] + assert ( + len(installed_distributions_by_key) > 1 + ), "We should resolve at least requests and urllib3" + assert "requests" in installed_distributions_by_key + assert Requirement.parse("urllib3==1.26.1") == installed_distributions_by_key["urllib3"] def test_pip_proprietary_url_with_markers_issues_1415(): - # https:///pip/ray-0.8.4-cp37-cp37m-linux_x86_64.whl; sys_platform == 'linux' - # ray==0.8.4; sys_platform == 'darwin' - resolved_dists = resolve_multi( + installed_dists = resolve( requirements=[ ( "https://files.pythonhosted.org/packages/53/18/" @@ -843,11 +847,11 @@ def test_pip_proprietary_url_with_markers_issues_1415(): "ansicolors==1.1.8; sys_platform == '{}'".format(sys.platform), ] ) - assert len(resolved_dists) == 1 + assert len(installed_dists) == 1 - resolved_dist = resolved_dists[0] - assert Requirement.parse("ansicolors==1.1.8") == resolved_dist.distribution.as_requirement() + installed_dist = installed_dists[0] + assert Requirement.parse("ansicolors==1.1.8") == installed_dist.distribution.as_requirement() assert ( Requirement.parse("ansicolors==1.1.8; sys_platform == '{}'".format(sys.platform)) - == resolved_dist.direct_requirement + == installed_dist.direct_requirement )