From 167909839a95ef5aa379fe12d4564b2b829cc175 Mon Sep 17 00:00:00 2001 From: Milo Minderbinder Date: Fri, 7 Jan 2022 17:35:02 -0500 Subject: [PATCH] fix TLS validation for requirements.txt Previously, due to a probable typo in the code for importing a requirements file to create a new pipenv project, SSL/TLS validation was disabled by default for any package index servers specified in the requirements file with the `--index-url` or `--extra-index-url` options. In addition, `--trusted-host` options in the requirements file would not work as intended, because any host or host:port pair provided with these options was incorrectly being matched against the full URLs of the configured index server(s) (i.e. including the scheme, path, etc.), instead of extracting and comparing with the host and port parts only, as intended. This PR fixes both of these issues, flipping the existing behavior to require SSL/TLS validation by default, and optionally allowing TLS validation to be disabled explicitly for specific host:port with the `--trusted-host` option if provided. --- pipenv/core.py | 13 +++++++++---- pipenv/utils.py | 24 ++++++++++++++++++++++++ tests/unit/test_utils.py | 2 +- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 420dbf5684..92811f746c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -17,7 +17,7 @@ from pipenv.patched import crayons from pipenv.utils import ( cmd_list_to_shell, convert_deps_to_pip, create_spinner, download_file, - find_python, get_canonical_names, get_source_list, is_pinned, + find_python, get_canonical_names, get_host_and_port, get_source_list, is_pinned, is_python_command, is_required_version, is_star, is_valid_url, parse_indexes, pep423_name, prepare_pip_source_args, proper_case, python_version, run_command, subprocess_run, venv_resolve_deps @@ -169,7 +169,7 @@ def import_requirements(project, r=None, dev=False): if extra_index: indexes.append(extra_index) if trusted_host: - trusted_hosts.append(trusted_host) + trusted_hosts.append(get_host_and_port(trusted_host)) indexes = sorted(set(indexes)) trusted_hosts = sorted(set(trusted_hosts)) reqs = [install_req_from_parsed_requirement(f) for f in parse_requirements(r, session=pip_requests)] @@ -185,8 +185,13 @@ def import_requirements(project, r=None, dev=False): else: project.add_package_to_pipfile(str(package.req), dev=dev) for index in indexes: - trusted = index in trusted_hosts - project.add_index_to_pipfile(index, verify_ssl=trusted) + # don't require HTTPS for trusted hosts (see: https://pip.pypa.io/en/stable/cli/pip/#cmdoption-trusted-host) + host_and_port = get_host_and_port(index) + require_valid_https = not any((v in trusted_hosts for v in ( + host_and_port, + host_and_port.partition(':')[0], # also check if hostname without port is in trusted_hosts + ))) + project.add_index_to_pipfile(index, verify_ssl=require_valid_https) project.recase_pipfile() diff --git a/pipenv/utils.py b/pipenv/utils.py index ac3b4ac975..0794df4636 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1643,6 +1643,30 @@ def get_url_name(url): return urllib3_util.parse_url(url).host +def get_host_and_port(url): + """Get the host, or the host:port pair if port is explicitly included, for the given URL. + + Examples: + >>> get_host_and_port('example.com') + 'example.com' + >>> get_host_and_port('example.com:443') + 'example.com:443' + >>> get_host_and_port('http://example.com') + 'example.com' + >>> get_host_and_port('https://example.com/') + 'example.com' + >>> get_host_and_port('https://example.com:8081') + 'example.com:8081' + >>> get_host_and_port('ssh://example.com') + 'example.com' + + :param url: the URL string to parse + :return: a string with the host:port pair if the URL includes port number explicitly; otherwise, returns host only + """ + url = urllib3_util.parse_url(url) + return '{}:{}'.format(url.host, url.port) if url.port else url.host + + def get_canonical_names(packages): """Canonicalize a list of packages and return a set of canonical names""" from .vendor.packaging.utils import canonicalize_name diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ec35502881..9139c8c360 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -142,7 +142,7 @@ def test_convert_deps_to_pip_unicode(): ("--extra-index-url=https://example.com/simple/", (None, "https://example.com/simple/", None, [])), ("--trusted-host=example.com", (None, None, "example.com", [])), ("# -i https://example.com/simple/", (None, None, None, [])), - ("requests", (None, None, None, ["requests"])) + ("requests # -i https://example.com/simple/", (None, None, None, ["requests"])), ]) @pytest.mark.utils def test_parse_indexes(line, result):