Fix locks for git+ssh with credentials.

Previously redacted credentials from the Pip download log were embedded in
locked artifact URLs rendering the lock unusable. Now credentials are fixed
up before the lock file is written.

Fixes pex-tool#1918

.github/workflows/ci.yml

@@ -158,6 +158,7 @@ jobs:
     name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-integration
     needs: org-check
     runs-on: ${{ matrix.os }}
+    environment: CI
     strategy:
       matrix:
         python-version: [[2, 7], [3, 10], [3, 11, "0-rc.2"]]
@@ -199,6 +200,10 @@ jobs:
         with:
           path: ${{ env._PEX_TEST_PYENV_ROOT }}
           key: ${{ runner.os }}-pyenv-root-v4
+      - name: Setup SSH Agent
+        uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
       - name: Run Integration Tests
         uses: pantsbuild/actions/run-tox@95209b287c817c78a765962d40ac6cea790fc511
         with:
@@ -207,6 +212,7 @@ jobs:
     name: (PyPy ${{ join(matrix.pypy-version, '.') }}) Pip ${{ matrix.pip-version }} TOXENV=pypy${{ join(matrix.pypy-version, '') }}-integration
     needs: org-check
     runs-on: ubuntu-20.04
+    environment: CI
     strategy:
       matrix:
         pypy-version: [[2, 7], [3, 9]]
@@ -245,6 +251,10 @@ jobs:
         with:
           path: ${{ env._PEX_TEST_PYENV_ROOT }}
           key: ${{ runner.os }}-pyenv-root-v4
+      - name: Setup SSH Agent
+        uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
       - name: Run Integration Tests
         uses: pantsbuild/actions/run-tox@95209b287c817c78a765962d40ac6cea790fc511
         with:

pex/requirements.py

@@ -695,15 +695,20 @@ def open_source():
             yield req_info
 
 
+def parse_requirement_string(requirement):
+    # type: (Text) -> ParsedRequirement
+    return _parse_requirement_line(
+        LogicalLine(
+            raw_text=requirement,
+            processed_text=requirement.strip(),
+            source="<string>",
+            start_line=1,
+            end_line=1,
+        )
+    )
+
+
 def parse_requirement_strings(requirements):
     # type: (Iterable[Text]) -> Iterator[ParsedRequirement]
     for requirement in requirements:
-        yield _parse_requirement_line(
-            LogicalLine(
-                raw_text=requirement,
-                processed_text=requirement.strip(),
-                source="<string>",
-                start_line=1,
-                end_line=1,
-            )
-        )
+        yield parse_requirement_string(requirement)

pex/resolve/downloads.py

@@ -11,6 +11,7 @@
 from pex.jobs import Job, Raise, SpawnedJob, execute_parallel
 from pex.pip.installation import get_pip
 from pex.pip.tool import PackageIndexConfiguration, Pip
+from pex.requirements import parse_requirement_string
 from pex.resolve import locker
 from pex.resolve.locked_resolve import Artifact, FileArtifact, LockConfiguration, LockStyle
 from pex.resolve.pep_691.fingerprint_service import FingerprintService
@@ -109,6 +110,7 @@ def _download(
         # observer does just this for universal locks with no target system or requires python
         # restrictions.
         download_observer = locker.patch(
+            root_requirements=[parse_requirement_string(url)],
             pip_version=self.package_index_configuration.pip_version,
             resolver=self.resolver,
             lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL),

pex/resolve/locker.py

@@ -8,6 +8,7 @@
 import os
 import pkgutil
 import re
+import sys
 from collections import defaultdict
 
 from pex.common import safe_mkdtemp
@@ -21,7 +22,7 @@
 from pex.pip.log_analyzer import LogAnalyzer
 from pex.pip.vcs import fingerprint_downloaded_vcs_archive
 from pex.pip.version import PipVersionValue
-from pex.requirements import VCS, VCSScheme, parse_scheme
+from pex.requirements import VCS, VCSRequirement, VCSScheme, parse_scheme
 from pex.resolve.locked_resolve import LockConfiguration, LockStyle, TargetSystem
 from pex.resolve.pep_691.fingerprint_service import FingerprintService
 from pex.resolve.pep_691.model import Endpoint
@@ -31,9 +32,22 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import DefaultDict, Dict, List, Optional, Pattern, Set, Text, Tuple
+    from typing import (
+        DefaultDict,
+        Dict,
+        Iterable,
+        List,
+        Mapping,
+        Optional,
+        Pattern,
+        Set,
+        Text,
+        Tuple,
+    )
 
     import attr  # vendor:skip
+
+    from pex.requirements import ParsedRequirement
 else:
     from pex.third_party import attr
 
@@ -50,16 +64,159 @@ class LockResult(object):
     local_projects = attr.ib()  # type: Tuple[str, ...]
 
 
+@attr.s(frozen=True)
+class Credentials(object):
+    username = attr.ib()  # type: str
+    password = attr.ib(default=None)  # type: Optional[str]
+
+    def are_redacted(self):
+        # type: () -> bool
+
+        # N.B.: Pip redacts here: pex/vendor/_vendored/pip/pip/_internal/utils/misc.py
+        return "****" in (self.username, self.password)
+
+    def render_basic_auth(self):
+        # type: () -> str
+        if self.password is not None:
+            return "{username}:{password}".format(username=self.username, password=self.password)
+        return self.username
+
+
+@attr.s(frozen=True)
+class Netloc(object):
+    host = attr.ib()  # type: str
+    port = attr.ib(default=None)  # type: Optional[int]
+
+    def render_host_port(self):
+        # type: () -> str
+        if self.port is not None:
+            return "{host}:{port}".format(host=self.host, port=self.port)
+        return self.host
+
+
+@attr.s(frozen=True)
+class CredentialedURL(object):
+    @classmethod
+    def parse(cls, url):
+        # type: (Text) -> CredentialedURL
+
+        url_info = urlparse.urlparse(url)
+
+        # The netloc component of the parsed url combines username, password, host and port. We need
+        # to track username and password; so we break up netloc into its four components.
+        credentials = (
+            Credentials(username=url_info.username, password=url_info.password)
+            if url_info.username
+            else None
+        )
+
+        netloc = Netloc(host=url_info.hostname, port=url_info.port) if url_info.hostname else None
+
+        return cls(
+            scheme=url_info.scheme,
+            credentials=credentials,
+            netloc=netloc,
+            path=url_info.path,
+            params=url_info.params,
+            query=url_info.query,
+            fragment=url_info.fragment,
+        )
+
+    scheme = attr.ib()  # type: str
+    credentials = attr.ib()  # type: Optional[Credentials]
+    netloc = attr.ib()  # type: Optional[Netloc]
+    path = attr.ib()  # type: str
+    params = attr.ib()  # type: str
+    query = attr.ib()  # type: str
+    fragment = attr.ib()  # type: str
+
+    @property
+    def has_redacted_credentials(self):
+        # type: () -> bool
+        if self.credentials is None:
+            return False
+        return self.credentials.are_redacted()
+
+    def strip_credentials(self):
+        # type: () -> CredentialedURL
+        return attr.evolve(self, credentials=None)
+
+    def strip_params_query_and_fragment(self):
+        # type: () -> CredentialedURL
+        return attr.evolve(self, params="", query="", fragment="")
+
+    def inject_credentials(self, credentials):
+        # type: (Optional[Credentials]) -> CredentialedURL
+        return attr.evolve(self, credentials=credentials)
+
+    def __str__(self):
+        # type: () -> str
+
+        netloc = ""
+        if self.netloc is not None:
+            host_port = self.netloc.render_host_port()
+            netloc = (
+                "{credentials}@{host_port}".format(
+                    credentials=self.credentials.render_basic_auth(), host_port=host_port
+                )
+                if self.credentials
+                else host_port
+            )
+
+        return urlparse.urlunparse(
+            (self.scheme, netloc, self.path, self.params, self.query, self.fragment)
+        )
+
+
+@attr.s(frozen=True)
+class VCSURLManager(object):
+    @staticmethod
+    def _normalize_vcs_url(credentialed_url):
+        # type: (CredentialedURL) -> str
+        return str(credentialed_url.strip_credentials().strip_params_query_and_fragment())
+
+    @classmethod
+    def create(cls, requirements):
+        # type: (Iterable[ParsedRequirement]) -> VCSURLManager
+
+        credentials_by_normalized_url = {}  # type: Dict[str, Optional[Credentials]]
+        for requirement in requirements:
+            if not isinstance(requirement, VCSRequirement):
+                continue
+            credentialed_url = CredentialedURL.parse(requirement.url)
+            vcs_url = "{vcs}+{url}".format(
+                vcs=requirement.vcs, url=cls._normalize_vcs_url(credentialed_url)
+            )
+            credentials_by_normalized_url[vcs_url] = credentialed_url.credentials
+        return cls(credentials_by_normalized_url)
+
+    _credentials_by_normalized_url = attr.ib()  # type: Mapping[str, Optional[Credentials]]
+
+    def normalize_url(self, url):
+        # type: (str) -> str
+
+        credentialed_url = CredentialedURL.parse(url)
+        if credentialed_url.has_redacted_credentials:
+            normalized_vcs_url = self._normalize_vcs_url(credentialed_url)
+            credentials = self._credentials_by_normalized_url.get(normalized_vcs_url)
+            if credentials is not None:
+                credentialed_url = credentialed_url.inject_credentials(credentials)
+        return str(credentialed_url)
+
+
 class Locker(LogAnalyzer):
     def __init__(
         self,
+        root_requirements,  # type: Iterable[ParsedRequirement]
         pip_version,  # type: PipVersionValue
         resolver,  # type: Resolver
         lock_configuration,  # type: LockConfiguration
         download_dir,  # type: str
         fingerprint_service=None,  # type: Optional[FingerprintService]
     ):
         # type: (...) -> None
+
+        self._vcs_url_manager = VCSURLManager.create(root_requirements)
         self._pip_version = pip_version
         self._resolver = resolver
         self._lock_configuration = lock_configuration
@@ -161,7 +318,7 @@ def analyze(self, line):
                                 project_name=requirement.project_name, version=Version(version)
                             ),
                             artifact=PartialArtifact(
-                                url=match.group("url"),
+                                url=self._vcs_url_manager.normalize_url(match.group("url")),
                                 fingerprint=fingerprint_downloaded_vcs_archive(
                                     download_dir=self._download_dir,
                                     project_name=str(requirement.project_name),
@@ -373,6 +530,7 @@ def lock_result(self):
 
 
 def patch(
+    root_requirements,  # type: Iterable[ParsedRequirement]
     pip_version,  # type: PipVersionValue
     resolver,  # type: Resolver
     lock_configuration,  # type: LockConfiguration
@@ -435,6 +593,7 @@ def patch(
 
     return DownloadObserver(
         analyzer=Locker(
+            root_requirements=root_requirements,
             pip_version=pip_version,
             resolver=resolver,
             lock_configuration=lock_configuration,

pex/resolve/lockfile/create.py

@@ -49,6 +49,7 @@
     import attr  # vendor:skip
 
     from pex.hashing import HintedDigest
+    from pex.requirements import ParsedRequirement
 else:
     from pex.third_party import attr
 
@@ -129,6 +130,7 @@ class _LockAnalysis(object):
 
 @attr.s(frozen=True)
 class LockObserver(ResolveObserver):
+    root_requirements = attr.ib()  # type: Tuple[ParsedRequirement, ...]
     lock_configuration = attr.ib()  # type: LockConfiguration
     resolver = attr.ib()  # type: Resolver
     wheel_builder = attr.ib()  # type: WheelBuilder
@@ -143,6 +145,7 @@ def observe_download(
     ):
         # type: (...) -> DownloadObserver
         patch = locker.patch(
+            root_requirements=self.root_requirements,
             pip_version=self.package_index_configuration.pip_version,
             resolver=self.resolver,
             lock_configuration=self.lock_configuration,
@@ -253,6 +256,7 @@ def create(
 
     configured_resolver = ConfiguredResolver(pip_configuration=pip_configuration)
     lock_observer = LockObserver(
+        root_requirements=parsed_requirements,
         lock_configuration=lock_configuration,
         resolver=configured_resolver,
         wheel_builder=WheelBuilder(