diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea1540236..a55015f25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,6 +158,7 @@ jobs: name: (${{ matrix.os }}) Pip ${{ matrix.pip-version }} TOXENV=py${{ matrix.python-version[0] }}${{ matrix.python-version[1] }}-integration needs: org-check runs-on: ${{ matrix.os }} + environment: CI strategy: matrix: python-version: [[2, 7], [3, 10], [3, 11, "0-rc.2"]] @@ -199,6 +200,10 @@ jobs: with: path: ${{ env._PEX_TEST_PYENV_ROOT }} key: ${{ runner.os }}-pyenv-root-v4 + - name: Setup SSH Agent + uses: webfactory/ssh-agent@v0.5.4 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - name: Run Integration Tests uses: pantsbuild/actions/run-tox@95209b287c817c78a765962d40ac6cea790fc511 with: @@ -207,6 +212,7 @@ jobs: name: (PyPy ${{ join(matrix.pypy-version, '.') }}) Pip ${{ matrix.pip-version }} TOXENV=pypy${{ join(matrix.pypy-version, '') }}-integration needs: org-check runs-on: ubuntu-20.04 + environment: CI strategy: matrix: pypy-version: [[2, 7], [3, 9]] @@ -245,6 +251,10 @@ jobs: with: path: ${{ env._PEX_TEST_PYENV_ROOT }} key: ${{ runner.os }}-pyenv-root-v4 + - name: Setup SSH Agent + uses: webfactory/ssh-agent@v0.5.4 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - name: Run Integration Tests uses: pantsbuild/actions/run-tox@95209b287c817c78a765962d40ac6cea790fc511 with: diff --git a/pex/requirements.py b/pex/requirements.py index 73f3ed002..e1015c931 100644 --- a/pex/requirements.py +++ b/pex/requirements.py @@ -695,15 +695,20 @@ def open_source(): yield req_info +def parse_requirement_string(requirement): + # type: (Text) -> ParsedRequirement + return _parse_requirement_line( + LogicalLine( + raw_text=requirement, + processed_text=requirement.strip(), + source="", + start_line=1, + end_line=1, + ) + ) + + def parse_requirement_strings(requirements): # type: (Iterable[Text]) -> Iterator[ParsedRequirement] for requirement in requirements: - yield _parse_requirement_line( - LogicalLine( - raw_text=requirement, - processed_text=requirement.strip(), - source="", - start_line=1, - end_line=1, - ) - ) + yield parse_requirement_string(requirement) diff --git a/pex/resolve/downloads.py b/pex/resolve/downloads.py index b3ddb0cf2..72003ea31 100644 --- a/pex/resolve/downloads.py +++ b/pex/resolve/downloads.py @@ -11,6 +11,7 @@ from pex.jobs import Job, Raise, SpawnedJob, execute_parallel from pex.pip.installation import get_pip from pex.pip.tool import PackageIndexConfiguration, Pip +from pex.requirements import parse_requirement_string from pex.resolve import locker from pex.resolve.locked_resolve import Artifact, FileArtifact, LockConfiguration, LockStyle from pex.resolve.pep_691.fingerprint_service import FingerprintService @@ -109,6 +110,7 @@ def _download( # observer does just this for universal locks with no target system or requires python # restrictions. download_observer = locker.patch( + root_requirements=[parse_requirement_string(url)], pip_version=self.package_index_configuration.pip_version, resolver=self.resolver, lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL), diff --git a/pex/resolve/locker.py b/pex/resolve/locker.py index d8b01a7ea..fb0af62c2 100644 --- a/pex/resolve/locker.py +++ b/pex/resolve/locker.py @@ -8,6 +8,7 @@ import os import pkgutil import re +import sys from collections import defaultdict from pex.common import safe_mkdtemp @@ -21,7 +22,7 @@ from pex.pip.log_analyzer import LogAnalyzer from pex.pip.vcs import fingerprint_downloaded_vcs_archive from pex.pip.version import PipVersionValue -from pex.requirements import VCS, VCSScheme, parse_scheme +from pex.requirements import VCS, VCSRequirement, VCSScheme, parse_scheme from pex.resolve.locked_resolve import LockConfiguration, LockStyle, TargetSystem from pex.resolve.pep_691.fingerprint_service import FingerprintService from pex.resolve.pep_691.model import Endpoint @@ -31,9 +32,22 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import DefaultDict, Dict, List, Optional, Pattern, Set, Text, Tuple + from typing import ( + DefaultDict, + Dict, + Iterable, + List, + Mapping, + Optional, + Pattern, + Set, + Text, + Tuple, + ) import attr # vendor:skip + + from pex.requirements import ParsedRequirement else: from pex.third_party import attr @@ -50,9 +64,150 @@ class LockResult(object): local_projects = attr.ib() # type: Tuple[str, ...] +@attr.s(frozen=True) +class Credentials(object): + username = attr.ib() # type: str + password = attr.ib(default=None) # type: Optional[str] + + def are_redacted(self): + # type: () -> bool + + # N.B.: Pip redacts here: pex/vendor/_vendored/pip/pip/_internal/utils/misc.py + return "****" in (self.username, self.password) + + def render_basic_auth(self): + # type: () -> str + if self.password is not None: + return "{username}:{password}".format(username=self.username, password=self.password) + return self.username + + +@attr.s(frozen=True) +class Netloc(object): + host = attr.ib() # type: str + port = attr.ib(default=None) # type: Optional[int] + + def render_host_port(self): + # type: () -> str + if self.port is not None: + return "{host}:{port}".format(host=self.host, port=self.port) + return self.host + + +@attr.s(frozen=True) +class CredentialedURL(object): + @classmethod + def parse(cls, url): + # type: (Text) -> CredentialedURL + + url_info = urlparse.urlparse(url) + + # The netloc component of the parsed url combines username, password, host and port. We need + # to track username and password; so we break up netloc into its four components. + credentials = ( + Credentials(username=url_info.username, password=url_info.password) + if url_info.username + else None + ) + + netloc = Netloc(host=url_info.hostname, port=url_info.port) if url_info.hostname else None + + return cls( + scheme=url_info.scheme, + credentials=credentials, + netloc=netloc, + path=url_info.path, + params=url_info.params, + query=url_info.query, + fragment=url_info.fragment, + ) + + scheme = attr.ib() # type: str + credentials = attr.ib() # type: Optional[Credentials] + netloc = attr.ib() # type: Optional[Netloc] + path = attr.ib() # type: str + params = attr.ib() # type: str + query = attr.ib() # type: str + fragment = attr.ib() # type: str + + @property + def has_redacted_credentials(self): + # type: () -> bool + if self.credentials is None: + return False + return self.credentials.are_redacted() + + def strip_credentials(self): + # type: () -> CredentialedURL + return attr.evolve(self, credentials=None) + + def strip_params_query_and_fragment(self): + # type: () -> CredentialedURL + return attr.evolve(self, params="", query="", fragment="") + + def inject_credentials(self, credentials): + # type: (Optional[Credentials]) -> CredentialedURL + return attr.evolve(self, credentials=credentials) + + def __str__(self): + # type: () -> str + + netloc = "" + if self.netloc is not None: + host_port = self.netloc.render_host_port() + netloc = ( + "{credentials}@{host_port}".format( + credentials=self.credentials.render_basic_auth(), host_port=host_port + ) + if self.credentials + else host_port + ) + + return urlparse.urlunparse( + (self.scheme, netloc, self.path, self.params, self.query, self.fragment) + ) + + +@attr.s(frozen=True) +class VCSURLManager(object): + @staticmethod + def _normalize_vcs_url(credentialed_url): + # type: (CredentialedURL) -> str + return str(credentialed_url.strip_credentials().strip_params_query_and_fragment()) + + @classmethod + def create(cls, requirements): + # type: (Iterable[ParsedRequirement]) -> VCSURLManager + + credentials_by_normalized_url = {} # type: Dict[str, Optional[Credentials]] + for requirement in requirements: + if not isinstance(requirement, VCSRequirement): + continue + credentialed_url = CredentialedURL.parse(requirement.url) + vcs_url = "{vcs}+{url}".format( + vcs=requirement.vcs, url=cls._normalize_vcs_url(credentialed_url) + ) + credentials_by_normalized_url[vcs_url] = credentialed_url.credentials + return cls(credentials_by_normalized_url) + + _credentials_by_normalized_url = attr.ib() # type: Mapping[str, Optional[Credentials]] + + def normalize_url(self, url): + # type: (str) -> str + + credentialed_url = CredentialedURL.parse(url) + if credentialed_url.has_redacted_credentials: + normalized_vcs_url = self._normalize_vcs_url(credentialed_url) + credentials = self._credentials_by_normalized_url.get(normalized_vcs_url) + if credentials is not None: + credentialed_url = credentialed_url.inject_credentials(credentials) + return str(credentialed_url) + + class Locker(LogAnalyzer): def __init__( self, + root_requirements, # type: Iterable[ParsedRequirement] pip_version, # type: PipVersionValue resolver, # type: Resolver lock_configuration, # type: LockConfiguration @@ -60,6 +215,8 @@ def __init__( fingerprint_service=None, # type: Optional[FingerprintService] ): # type: (...) -> None + + self._vcs_url_manager = VCSURLManager.create(root_requirements) self._pip_version = pip_version self._resolver = resolver self._lock_configuration = lock_configuration @@ -161,7 +318,7 @@ def analyze(self, line): project_name=requirement.project_name, version=Version(version) ), artifact=PartialArtifact( - url=match.group("url"), + url=self._vcs_url_manager.normalize_url(match.group("url")), fingerprint=fingerprint_downloaded_vcs_archive( download_dir=self._download_dir, project_name=str(requirement.project_name), @@ -373,6 +530,7 @@ def lock_result(self): def patch( + root_requirements, # type: Iterable[ParsedRequirement] pip_version, # type: PipVersionValue resolver, # type: Resolver lock_configuration, # type: LockConfiguration @@ -435,6 +593,7 @@ def patch( return DownloadObserver( analyzer=Locker( + root_requirements=root_requirements, pip_version=pip_version, resolver=resolver, lock_configuration=lock_configuration, diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 0c41d8257..06ffb21bd 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -49,6 +49,7 @@ import attr # vendor:skip from pex.hashing import HintedDigest + from pex.requirements import ParsedRequirement else: from pex.third_party import attr @@ -129,6 +130,7 @@ class _LockAnalysis(object): @attr.s(frozen=True) class LockObserver(ResolveObserver): + root_requirements = attr.ib() # type: Tuple[ParsedRequirement, ...] lock_configuration = attr.ib() # type: LockConfiguration resolver = attr.ib() # type: Resolver wheel_builder = attr.ib() # type: WheelBuilder @@ -143,6 +145,7 @@ def observe_download( ): # type: (...) -> DownloadObserver patch = locker.patch( + root_requirements=self.root_requirements, pip_version=self.package_index_configuration.pip_version, resolver=self.resolver, lock_configuration=self.lock_configuration, @@ -253,6 +256,7 @@ def create( configured_resolver = ConfiguredResolver(pip_configuration=pip_configuration) lock_observer = LockObserver( + root_requirements=parsed_requirements, lock_configuration=lock_configuration, resolver=configured_resolver, wheel_builder=WheelBuilder( diff --git a/tests/integration/resolve/test_issue_1918.py b/tests/integration/resolve/test_issue_1918.py new file mode 100644 index 000000000..2575aa981 --- /dev/null +++ b/tests/integration/resolve/test_issue_1918.py @@ -0,0 +1,107 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +import itertools +import os.path + +import pytest + +from pex.cli.testing import run_pex3 +from pex.compatibility import commonpath +from pex.dist_metadata import Requirement +from pex.pip.version import PipVersion, PipVersionValue +from pex.requirements import VCS +from pex.resolve.locked_resolve import VCSArtifact +from pex.resolve.lockfile import json_codec +from pex.resolve.resolver_configuration import ResolverVersion +from pex.sorted_tuple import SortedTuple +from pex.testing import run_pex_command +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + + +VCS_URL = ( + "git+ssh://git@github.com/jonathaneunice/colors.git@c965f5b9103c5bd32a1572adb8024ebe83278fb0" +) + + +@pytest.mark.parametrize( + ["requirement", "expected_url"], + [ + pytest.param( + "ansicolors @ {vcs_url}".format(vcs_url=VCS_URL), VCS_URL, id="direct-reference" + ), + pytest.param( + *itertools.repeat("{vcs_url}#egg=ansicolors".format(vcs_url=VCS_URL), 2), + id="pip-proprietary" + ), + ], +) +@pytest.mark.parametrize( + "pip_version", + [pytest.param(pip_version, id=pip_version.value) for pip_version in PipVersion.values()], +) +@pytest.mark.parametrize( + "resolver_version", + [ + pytest.param(resolver_version, id=resolver_version.value) + for resolver_version in ResolverVersion.values() + ], +) +def test_redacted_requirement_handling( + tmpdir, # type: Any + requirement, # type: str + expected_url, # type: str + pip_version, # type: PipVersionValue + resolver_version, # type: ResolverVersion.Value +): + # type: (...) -> None + + lock = os.path.join(str(tmpdir), "lock.json") + run_pex3( + "lock", + "create", + "--pip-version", + str(pip_version), + "--resolver-version", + str(resolver_version), + requirement, + "-o", + lock, + "--indent", + "2", + ).assert_success() + lockfile = json_codec.load(lock) + assert SortedTuple([Requirement.parse("ansicolors")]) == lockfile.requirements + + assert 1 == len(lockfile.locked_resolves) + locked_resolve = lockfile.locked_resolves[0] + + assert 1 == len(locked_resolve.locked_requirements) + locked_requirement = locked_resolve.locked_requirements[0] + + artifacts = list(locked_requirement.iter_artifacts()) + assert 1 == len(artifacts) + artifact = artifacts[0] + + assert isinstance(artifact, VCSArtifact) + assert VCS.Git is artifact.vcs + assert expected_url == artifact.url + + pex_root = os.path.join(str(tmpdir), "pex_root") + result = run_pex_command( + args=[ + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + "--lock", + lock, + "--", + "-c", + "import colors; print(colors.__file__)", + ] + ) + result.assert_success() + assert pex_root == commonpath([pex_root, result.output.strip()]) diff --git a/tests/integration/test_locked_resolve.py b/tests/integration/test_locked_resolve.py index 19b53e4a5..0cc8079cf 100644 --- a/tests/integration/test_locked_resolve.py +++ b/tests/integration/test_locked_resolve.py @@ -50,6 +50,7 @@ def create_lock_observer(lock_configuration): network_configuration=pip_configuration.network_configuration, ) return LockObserver( + root_requirements=(), lock_configuration=lock_configuration, resolver=ConfiguredResolver(pip_configuration=pip_configuration), wheel_builder=WheelBuilder( diff --git a/tests/resolve/test_locker.py b/tests/resolve/test_locker.py new file mode 100644 index 000000000..71d01d55b --- /dev/null +++ b/tests/resolve/test_locker.py @@ -0,0 +1,85 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pex.resolve.locker import CredentialedURL, Credentials, Netloc +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Optional + + +def test_credentials_redacted(): + # type: () -> None + + assert not Credentials("git").are_redacted() + assert not Credentials("git", "").are_redacted() + assert not Credentials("joe", "bob").are_redacted() + + assert Credentials("****").are_redacted() + assert Credentials("joe", "****").are_redacted() + + +def test_basic_auth_rendering(): + # type: () -> None + + assert "git" == Credentials("git").render_basic_auth() + assert "git:" == Credentials("git", "").render_basic_auth() + assert "joe:bob" == Credentials("joe", "bob").render_basic_auth() + + +def test_host_port_rendering(): + # type: () -> None + + assert "example.com" == Netloc("example.com").render_host_port() + assert "example.com:80" == Netloc("example.com", 80).render_host_port() + + +def test_strip_credentials(): + # type: () -> None + + def strip_credentials( + url, # type: str + expected_credentials=None, # type: Optional[Credentials] + ): + # type: (...) -> str + credentialed_url = CredentialedURL.parse(url) + assert expected_credentials == credentialed_url.credentials + return str(credentialed_url.strip_credentials()) + + assert "file:///a/file" == strip_credentials("file:///a/file") + assert "http://example.com" == strip_credentials("http://example.com") + assert "http://example.com:80" == strip_credentials("http://example.com:80") + assert "http://example.com" == strip_credentials("http://joe@example.com", Credentials("joe")) + assert "https://example.com:443" == strip_credentials( + "https://joe@example.com:443", Credentials("joe") + ) + assert "http://example.com" == strip_credentials( + "http://joe:bob@example.com", Credentials("joe", "bob") + ) + assert "phys://example.com:1137" == strip_credentials( + "phys://joe:bob@example.com:1137", Credentials("joe", "bob") + ) + assert "git://example.org" == strip_credentials("git://git@example.org", Credentials("git")) + assert "git://example.org" == strip_credentials("git://****@example.org", Credentials("****")) + + +def test_inject_credentials(): + # type: () -> None + + def inject_credentials( + url, # type: str + credentials, # type: Optional[Credentials] + ): + # type: (...) -> str + return str(CredentialedURL.parse(url).inject_credentials(credentials)) + + assert "git://git@example.org" == inject_credentials("git://example.org", Credentials("git")) + assert "git://git@example.org" == inject_credentials( + "git://****@example.org", Credentials("git") + ) + assert "http://joe:bob@example.org" == inject_credentials( + "http://example.org", Credentials("joe", "bob") + ) + assert "http://joe:bob@example.org" == inject_credentials( + "http://joe:****@example.org", Credentials("joe", "bob") + ) diff --git a/tox.ini b/tox.ini index 622d97cb8..a6a1fdff0 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,8 @@ passenv = PATHEXT USER USERNAME + # Needed for tests of git+ssh://... + SSH_AUTH_SOCK setenv = pip20: _PEX_PIP_VERSION=20.3.4-patched pip22: _PEX_PIP_VERSION=22.2.2