Merge pip 20.3.4 commits. (#9)

* Merge pull request pypa#9289 from uranusjr/new-resolver-lazy-insert

* Merge pull request pypa#9315 from pradyunsg/better-search-errors

* Merge pull request pypa#9369 from uranusjr/resolvelib-054

* Merge pull request pypa#9320 from uranusjr/wheel-check-valid

Verify built wheel contains valid metadata

* Merge pull request pypa#9432 from uranusjr/new-resolver-dedup-on-backtracking

Avoid downloading multiple candidates of a same version

* Revert "Remove on_returncode parameter from call_subprocess"

This reverts commit ab3ee71.

* Revert "Remove show_stdout from run_command args"

This reverts commit 94882fd.

* Revert "Create call_subprocess just for vcs commands"

This reverts commit 8adbc21.

* Revert "Improve check for svn version string"

This reverts commit 1471897.

* Revert "Bubble up SubProcessError to basecommand._main"

This reverts commit e9f738a.

* Additional revert of 7969

Revert additional changes that were made
after 7969 and depended on it.

* add stdout_only to call_subprocess

* vcs: capture subprocess stdout only

* Add test for call_subprocess stdout_only

* Bump for release

* Fix test bitrot.

Setuptools released 52.0.0 which killed easy_install support and thus
caused tests exercising easy_install to fail.

Co-authored-by: Pradyun Gedam <[email protected]>
Co-authored-by: Stéphane Bidoul <[email protected]>
Co-authored-by: Pradyun Gedam <[email protected]>

NEWS.rst

@@ -9,6 +9,39 @@
 
 .. towncrier release notes start
 
+20.3.4 (2021-01-23)
+===================
+
+Features
+--------
+
+- ``pip wheel`` now verifies the built wheel contains valid metadata, and can be
+  installed by a subsequent ``pip install``. This can be disabled with
+  ``--no-verify``. (`#9206 <https://github.com/pypa/pip/issues/9206>`_)
+- Improve presentation of XMLRPC errors in pip search. (`#9315 <https://github.com/pypa/pip/issues/9315>`_)
+
+Bug Fixes
+---------
+
+- Fixed hanging VCS subprocess calls when the VCS outputs a large amount of data
+  on stderr. Restored logging of VCS errors that was inadvertently removed in pip
+  20.2. (`#8876 <https://github.com/pypa/pip/issues/8876>`_)
+- Fix error when an existing incompatibility is unable to be applied to a backtracked state. (`#9180 <https://github.com/pypa/pip/issues/9180>`_)
+- New resolver: Discard a faulty distribution, instead of quitting outright.
+  This implementation is taken from 20.2.2, with a fix that always makes the
+  resolver iterate through candidates from indexes lazily, to avoid downloading
+  candidates we do not need. (`#9203 <https://github.com/pypa/pip/issues/9203>`_)
+- New resolver: Discard a source distribution if it fails to generate metadata,
+  instead of quitting outright. This implementation is taken from 20.2.2, with a
+  fix that always makes the resolver iterate through candidates from indexes
+  lazily, to avoid downloading candidates we do not need. (`#9246 <https://github.com/pypa/pip/issues/9246>`_)
+
+Vendored Libraries
+------------------
+
+- Upgrade resolvelib to 0.5.4.
+
+
 20.3.3 (2020-12-15)
 ===================
 

src/pip/__init__.py

@@ -4,7 +4,7 @@
     from typing import List, Optional
 
 
-__version__ = "21.0.dev0"
+__version__ = "20.3.4"
 
 
 def main(args=None):

src/pip/_internal/cli/base_command.py

@@ -27,7 +27,6 @@
     InstallationError,
     NetworkConnectionError,
     PreviousBuildDirError,
-    SubProcessError,
     UninstallationError,
 )
 from pip._internal.utils.deprecation import deprecated
@@ -230,7 +229,7 @@ def _main(self, args):
 
             return PREVIOUS_BUILD_DIR_ERROR
         except (InstallationError, UninstallationError, BadCommand,
-                SubProcessError, NetworkConnectionError) as exc:
+                NetworkConnectionError) as exc:
             logger.critical(str(exc))
             logger.debug('Exception information:', exc_info=True)
 

src/pip/_internal/commands/install.py

@@ -347,6 +347,7 @@ def run(self, options, args):
             _, build_failures = build(
                 reqs_to_build,
                 wheel_cache=wheel_cache,
+                verify=True,
                 build_options=[],
                 global_options=[],
             )

src/pip/_internal/commands/search.py

@@ -79,7 +79,14 @@ def search(self, query, options):
 
         transport = PipXmlrpcTransport(index_url, session)
         pypi = xmlrpc_client.ServerProxy(index_url, transport)
-        hits = pypi.search({'name': query, 'summary': query}, 'or')
+        try:
+            hits = pypi.search({'name': query, 'summary': query}, 'or')
+        except xmlrpc_client.Fault as fault:
+            message = "XMLRPC request failed [code: {code}]\n{string}".format(
+                code=fault.faultCode,
+                string=fault.faultString,
+            )
+            raise CommandError(message)
         return hits
 
 

src/pip/_internal/commands/wheel.py

@@ -81,6 +81,14 @@ def add_options(self):
         self.cmd_opts.add_option(cmdoptions.build_dir())
         self.cmd_opts.add_option(cmdoptions.progress_bar())
 
+        self.cmd_opts.add_option(
+            '--no-verify',
+            dest='no_verify',
+            action='store_true',
+            default=False,
+            help="Don't verify if built wheel is valid.",
+        )
+
         self.cmd_opts.add_option(
             '--global-option',
             dest='global_options',
@@ -166,6 +174,7 @@ def run(self, options, args):
         build_successes, build_failures = build(
             reqs_to_build,
             wheel_cache=wheel_cache,
+            verify=(not options.no_verify),
             build_options=options.build_options or [],
             global_options=options.global_options or [],
         )

src/pip/_internal/exceptions.py

@@ -91,11 +91,6 @@ class CommandError(PipError):
     """Raised when there is an error in command-line arguments"""
 
 
-class SubProcessError(PipError):
-    """Raised when there is an error raised while executing a
-    command in subprocess"""
-
-
 class PreviousBuildDirError(PipError):
     """Raised when there's a previous conflicting build directory"""
 
@@ -151,6 +146,21 @@ def __str__(self):
         )
 
 
+class InstallationSubprocessError(InstallationError):
+    """A subprocess call failed during installation."""
+    def __init__(self, returncode, description):
+        # type: (int, str) -> None
+        self.returncode = returncode
+        self.description = description
+
+    def __str__(self):
+        # type: () -> str
+        return (
+            "Command errored out with exit status {}: {} "
+            "Check the logs for full command output."
+        ).format(self.returncode, self.description)
+
+
 class HashErrors(InstallationError):
     """Multiple HashError instances rolled into one for reporting"""
 

src/pip/_internal/resolution/resolvelib/candidates.py

@@ -141,7 +141,7 @@ def __init__(
         self._ireq = ireq
         self._name = name
         self._version = version
-        self._dist = None  # type: Optional[Distribution]
+        self.dist = self._prepare()
 
     def __str__(self):
         # type: () -> str
@@ -209,8 +209,6 @@ def _prepare_distribution(self):
     def _check_metadata_consistency(self, dist):
         # type: (Distribution) -> None
         """Check for consistency of project name and version of dist."""
-        # TODO: (Longer term) Rather than abort, reject this candidate
-        #       and backtrack. This would need resolvelib support.
         name = canonicalize_name(dist.project_name)
         if self._name is not None and self._name != name:
             raise MetadataInconsistent(self._ireq, "name", dist.project_name)
@@ -219,25 +217,17 @@ def _check_metadata_consistency(self, dist):
             raise MetadataInconsistent(self._ireq, "version", dist.version)
 
     def _prepare(self):
-        # type: () -> None
-        if self._dist is not None:
-            return
+        # type: () -> Distribution
         try:
             dist = self._prepare_distribution()
         except HashError as e:
+            # Provide HashError the underlying ireq that caused it. This
+            # provides context for the resulting error message to show the
+            # offending line to the user.
             e.req = self._ireq
             raise
-
-        assert dist is not None, "Distribution already installed"
         self._check_metadata_consistency(dist)
-        self._dist = dist
-
-    @property
-    def dist(self):
-        # type: () -> Distribution
-        if self._dist is None:
-            self._prepare()
-        return self._dist
+        return dist
 
     def _get_requires_python_dependency(self):
         # type: () -> Optional[Requirement]
@@ -261,7 +251,6 @@ def iter_dependencies(self, with_requires):
 
     def get_install_requirement(self):
         # type: () -> Optional[InstallRequirement]
-        self._prepare()
         return self._ireq
 
 

src/pip/_internal/resolution/resolvelib/factory.py

@@ -5,6 +5,8 @@
 from pip._internal.exceptions import (
     DistributionNotFound,
     InstallationError,
+    InstallationSubprocessError,
+    MetadataInconsistent,
     UnsupportedPythonVersion,
     UnsupportedWheel,
 )
@@ -33,6 +35,7 @@
     ExplicitRequirement,
     RequiresPythonRequirement,
     SpecifierRequirement,
+    UnsatisfiableRequirement,
 )
 
 if MYPY_CHECK_RUNNING:
@@ -94,6 +97,7 @@ def __init__(
         self._force_reinstall = force_reinstall
         self._ignore_requires_python = ignore_requires_python
 
+        self._build_failures = {}  # type: Cache[InstallationError]
         self._link_candidate_cache = {}  # type: Cache[LinkCandidate]
         self._editable_candidate_cache = {}  # type: Cache[EditableCandidate]
         self._installed_candidate_cache = {
@@ -136,21 +140,40 @@ def _make_candidate_from_link(
         name,  # type: Optional[str]
         version,  # type: Optional[_BaseVersion]
     ):
-        # type: (...) -> Candidate
+        # type: (...) -> Optional[Candidate]
         # TODO: Check already installed candidate, and use it if the link and
         # editable flag match.
+
+        if link in self._build_failures:
+            # We already tried this candidate before, and it does not build.
+            # Don't bother trying again.
+            return None
+
         if template.editable:
             if link not in self._editable_candidate_cache:
-                self._editable_candidate_cache[link] = EditableCandidate(
-                    link, template, factory=self, name=name, version=version,
-                )
+                try:
+                    self._editable_candidate_cache[link] = EditableCandidate(
+                        link, template, factory=self,
+                        name=name, version=version,
+                    )
+                except (InstallationSubprocessError, MetadataInconsistent) as e:
+                    logger.warning("Discarding %s. %s", link, e)
+                    self._build_failures[link] = e
+                    return None
             base = self._editable_candidate_cache[link]  # type: BaseCandidate
         else:
             if link not in self._link_candidate_cache:
-                self._link_candidate_cache[link] = LinkCandidate(
-                    link, template, factory=self, name=name, version=version,
-                )
+                try:
+                    self._link_candidate_cache[link] = LinkCandidate(
+                        link, template, factory=self,
+                        name=name, version=version,
+                    )
+                except (InstallationSubprocessError, MetadataInconsistent) as e:
+                    logger.warning("Discarding %s. %s", link, e)
+                    self._build_failures[link] = e
+                    return None
             base = self._link_candidate_cache[link]
+
         if extras:
             return ExtrasCandidate(base, extras)
         return base
@@ -207,16 +230,23 @@ def iter_index_candidates():
             all_yanked = all(ican.link.is_yanked for ican in icans)
 
             # PackageFinder returns earlier versions first, so we reverse.
+            versions_found = set()  # type: Set[_BaseVersion]
             for ican in reversed(icans):
                 if not all_yanked and ican.link.is_yanked:
                     continue
-                yield self._make_candidate_from_link(
+                if ican.version in versions_found:
+                    continue
+                candidate = self._make_candidate_from_link(
                     link=ican.link,
                     extras=extras,
                     template=template,
                     name=name,
                     version=ican.version,
                 )
+                if candidate is None:
+                    continue
+                yield candidate
+                versions_found.add(ican.version)
 
         return FoundCandidates(
             iter_index_candidates,
@@ -280,6 +310,16 @@ def make_requirement_from_install_req(self, ireq, requested_extras):
             name=canonicalize_name(ireq.name) if ireq.name else None,
             version=None,
         )
+        if cand is None:
+            # There's no way we can satisfy a URL requirement if the underlying
+            # candidate fails to build. An unnamed URL must be user-supplied, so
+            # we fail eagerly. If the URL is named, an unsatisfiable requirement
+            # can make the resolver do the right thing, either backtrack (and
+            # maybe find some other requirement that's buildable) or raise a
+            # ResolutionImpossible eventually.
+            if not ireq.name:
+                raise self._build_failures[ireq.link]
+            return UnsatisfiableRequirement(canonicalize_name(ireq.name))
         return self.make_requirement_from_candidate(cand)
 
     def make_requirement_from_candidate(self, candidate):