From 761b2064f42e6904bba189d89b29100f7c03ca1f Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Fri, 21 Jun 2019 08:11:26 -0700 Subject: [PATCH 1/2] Remove models/search_scope.py's dependency on download.py. --- src/pip/_internal/download.py | 11 ++--------- src/pip/_internal/models/search_scope.py | 2 +- src/pip/_internal/utils/compat.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 0aa692b150b..f57d4734caf 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -25,12 +25,13 @@ from pip._vendor.six.moves import xmlrpc_client # type: ignore from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._vendor.six.moves.urllib import request as urllib_request -from pip._vendor.urllib3.util import IS_PYOPENSSL import pip from pip._internal.exceptions import HashMismatch, InstallationError from pip._internal.locations import write_delete_marker_file from pip._internal.models.index import PyPI +# Import ssl from compat so the initial import occurs in only one place. +from pip._internal.utils.compat import HAS_TLS, ssl from pip._internal.utils.encoding import auto_decode from pip._internal.utils.filesystem import check_path_owner from pip._internal.utils.glibc import libc_ver @@ -54,14 +55,6 @@ from pip._internal.utils.hashes import Hashes from pip._internal.vcs.versioncontrol import AuthInfo, VersionControl -try: - import ssl # noqa -except ImportError: - ssl = None - - -HAS_TLS = (ssl is not None) or IS_PYOPENSSL - __all__ = ['get_file_content', 'is_url', 'url_to_path', 'path_to_url', 'is_archive_file', 'unpack_vcs_link', diff --git a/src/pip/_internal/models/search_scope.py b/src/pip/_internal/models/search_scope.py index c570586d2d7..62152449540 100644 --- a/src/pip/_internal/models/search_scope.py +++ b/src/pip/_internal/models/search_scope.py @@ -6,8 +6,8 @@ from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._internal.download import HAS_TLS from pip._internal.models.index import PyPI +from pip._internal.utils.compat import HAS_TLS from pip._internal.utils.misc import normalize_path, redact_password_from_url from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py index 845436e4894..0250c07ecb5 100644 --- a/src/pip/_internal/utils/compat.py +++ b/src/pip/_internal/utils/compat.py @@ -10,12 +10,21 @@ import sys from pip._vendor.six import text_type +from pip._vendor.urllib3.util import IS_PYOPENSSL from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Tuple, Text +try: + import _ssl # noqa +except ImportError: + ssl = None +else: + # This additional assignment was needed to prevent a mypy error. + ssl = _ssl + try: import ipaddress except ImportError: @@ -36,6 +45,8 @@ logger = logging.getLogger(__name__) +HAS_TLS = (ssl is not None) or IS_PYOPENSSL + if sys.version_info >= (3, 4): uses_pycache = True from importlib.util import cache_from_source From cab7316e2b6f994d95527c0577c14f6ae2164126 Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Fri, 21 Jun 2019 09:10:03 -0700 Subject: [PATCH 2/2] Change PackageFinder.create() to accept a SearchScope object. --- src/pip/_internal/cli/base_command.py | 16 ++++---------- src/pip/_internal/cli/cmdoptions.py | 26 ++++++++++++++++++++++ src/pip/_internal/commands/list.py | 15 ++++++------- src/pip/_internal/commands/wheel.py | 5 ----- src/pip/_internal/index.py | 10 ++------- src/pip/_internal/utils/outdated.py | 6 ++++-- tests/lib/__init__.py | 9 ++++++-- tests/unit/test_build_env.py | 4 +++- tests/unit/test_cmdoptions.py | 31 ++++++++++++++++++++++++++- tests/unit/test_index.py | 11 ++++++++-- tests/unit/test_unit_outdated.py | 4 ++-- 11 files changed, 93 insertions(+), 44 deletions(-) diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 31e5f6afedc..78a4e1160e7 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -10,6 +10,7 @@ import traceback from pip._internal.cli import cmdoptions +from pip._internal.cli.cmdoptions import make_search_scope from pip._internal.cli.parser import ( ConfigOptionParser, UpdatingDefaultsHelpFormatter, ) @@ -31,9 +32,7 @@ from pip._internal.req.req_file import parse_requirements from pip._internal.utils.deprecation import deprecated from pip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging -from pip._internal.utils.misc import ( - get_prog, normalize_path, redact_password_from_url, -) +from pip._internal.utils.misc import get_prog, normalize_path from pip._internal.utils.outdated import pip_version_check from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -337,13 +336,7 @@ def _build_package_finder( :param ignore_requires_python: Whether to ignore incompatible "Requires-Python" values in links. Defaults to False. """ - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.debug( - 'Ignoring indexes: %s', - ','.join(redact_password_from_url(url) for url in index_urls), - ) - index_urls = [] + search_scope = make_search_scope(options) target_python = TargetPython( platform=platform, @@ -353,9 +346,8 @@ def _build_package_finder( ) return PackageFinder.create( - find_links=options.find_links, + search_scope=search_scope, format_control=options.format_control, - index_urls=index_urls, trusted_hosts=options.trusted_hosts, allow_all_prereleases=options.pre, session=session, diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 4b766b91871..eba545c8509 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -9,6 +9,7 @@ """ from __future__ import absolute_import +import logging import textwrap import warnings from distutils.util import strtobool @@ -20,7 +21,9 @@ from pip._internal.locations import USER_CACHE_DIR, src_prefix from pip._internal.models.format_control import FormatControl from pip._internal.models.index import PyPI +from pip._internal.models.search_scope import SearchScope from pip._internal.utils.hashes import STRONG_HASHES +from pip._internal.utils.misc import redact_password_from_url from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import BAR_TYPES @@ -29,6 +32,8 @@ from optparse import OptionParser, Values from pip._internal.cli.parser import ConfigOptionParser +logger = logging.getLogger(__name__) + def raise_option_error(parser, option, msg): """ @@ -350,6 +355,27 @@ def find_links(): ) +def make_search_scope(options, suppress_no_index=False): + """ + :param suppress_no_index: Whether to ignore the --no-index option + when constructing the SearchScope object. + """ + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index and not suppress_no_index: + logger.debug( + 'Ignoring indexes: %s', + ','.join(redact_password_from_url(url) for url in index_urls), + ) + index_urls = [] + + search_scope = SearchScope( + find_links=options.find_links, + index_urls=index_urls, + ) + + return search_scope + + def trusted_host(): # type: () -> Option return Option( diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index add31c0bff8..42dd1fc3cff 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -8,6 +8,7 @@ from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command +from pip._internal.cli.cmdoptions import make_search_scope from pip._internal.exceptions import CommandError from pip._internal.index import PackageFinder from pip._internal.utils.misc import ( @@ -109,13 +110,14 @@ def __init__(self, *args, **kw): self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, cmd_opts) - def _build_package_finder(self, options, index_urls, session): + def _build_package_finder(self, options, session): """ Create a package finder appropriate to this list command. """ + search_scope = make_search_scope(options) + return PackageFinder.create( - find_links=options.find_links, - index_urls=index_urls, + search_scope=search_scope, allow_all_prereleases=options.pre, trusted_hosts=options.trusted_hosts, session=session, @@ -169,13 +171,8 @@ def get_not_required(self, packages, options): return {pkg for pkg in packages if pkg.key not in dep_keys} def iter_packages_latest_infos(self, packages, options): - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.debug('Ignoring indexes: %s', ','.join(index_urls)) - index_urls = [] - with self._build_session(options) as session: - finder = self._build_package_finder(options, index_urls, session) + finder = self._build_package_finder(options, session) for dist in packages: typ = 'unknown' diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 168ca12f64f..97f3b148af5 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -108,11 +108,6 @@ def __init__(self, *args, **kw): def run(self, options, args): cmdoptions.check_install_build_global(options) - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.debug('Ignoring indexes: %s', ','.join(index_urls)) - index_urls = [] - if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index 7bc20e758ef..f58a318b216 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -26,7 +26,6 @@ from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.format_control import FormatControl from pip._internal.models.link import Link -from pip._internal.models.search_scope import SearchScope from pip._internal.models.target_python import TargetPython from pip._internal.utils.compat import ipaddress from pip._internal.utils.logging import indent_log @@ -46,6 +45,7 @@ ) from pip._vendor.packaging.version import _BaseVersion from pip._vendor.requests import Response + from pip._internal.models.search_scope import SearchScope from pip._internal.req import InstallRequirement from pip._internal.download import PipSession @@ -592,8 +592,7 @@ def __init__( @classmethod def create( cls, - find_links, # type: List[str] - index_urls, # type: List[str] + search_scope, # type: SearchScope allow_all_prereleases=False, # type: bool trusted_hosts=None, # type: Optional[List[str]] session=None, # type: Optional[PipSession] @@ -623,11 +622,6 @@ def create( "'session'" ) - search_scope = SearchScope.create( - find_links=find_links, - index_urls=index_urls, - ) - candidate_evaluator = CandidateEvaluator( target_python=target_python, prefer_binary=prefer_binary, diff --git a/src/pip/_internal/utils/outdated.py b/src/pip/_internal/utils/outdated.py index 3b3aa9a3eb6..af0b9b497fe 100644 --- a/src/pip/_internal/utils/outdated.py +++ b/src/pip/_internal/utils/outdated.py @@ -9,6 +9,7 @@ from pip._vendor import lockfile, pkg_resources from pip._vendor.packaging import version as packaging_version +from pip._internal.cli.cmdoptions import make_search_scope from pip._internal.index import PackageFinder from pip._internal.utils.compat import WINDOWS from pip._internal.utils.filesystem import check_path_owner @@ -122,9 +123,10 @@ def pip_version_check(session, options): # Refresh the version if we need to or just see if we need to warn if pypi_version is None: # Lets use PackageFinder to see what the latest pip version is + search_scope = make_search_scope(options, suppress_no_index=True) + finder = PackageFinder.create( - find_links=options.find_links, - index_urls=[options.index_url] + options.extra_index_urls, + search_scope=search_scope, allow_all_prereleases=False, # Explicitly set to False trusted_hosts=options.trusted_hosts, session=session, diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index bbdb02307e0..ac706b4afb7 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -13,6 +13,7 @@ import pytest from scripttest import FoundDir, TestFileEnvironment +from pip._internal.models.search_scope import SearchScope from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX from pip._internal.download import PipSession from pip._internal.index import PackageFinder @@ -96,9 +97,13 @@ def make_test_finder( if session is None: session = PipSession() + search_scope = SearchScope.create( + find_links=find_links, + index_urls=index_urls, + ) + return PackageFinder.create( - find_links, - index_urls, + search_scope=search_scope, allow_all_prereleases=allow_all_prereleases, trusted_hosts=trusted_hosts, session=session, diff --git a/tests/unit/test_build_env.py b/tests/unit/test_build_env.py index 5126024eb16..4f279ff8f82 100644 --- a/tests/unit/test_build_env.py +++ b/tests/unit/test_build_env.py @@ -24,8 +24,10 @@ def run_with_build_env(script, setup_script_contents, from pip._internal.build_env import BuildEnvironment from pip._internal.download import PipSession from pip._internal.index import PackageFinder + from pip._internal.models.search_scope import SearchScope - finder = PackageFinder.create([%r], [], session=PipSession()) + search_scope = SearchScope.create([%r], []) + finder = PackageFinder.create(search_scope, session=PipSession()) build_env = BuildEnvironment() try: diff --git a/tests/unit/test_cmdoptions.py b/tests/unit/test_cmdoptions.py index 150570e716e..2c72244fdd7 100644 --- a/tests/unit/test_cmdoptions.py +++ b/tests/unit/test_cmdoptions.py @@ -1,6 +1,35 @@ +import pretend import pytest -from pip._internal.cli.cmdoptions import _convert_python_version +from pip._internal.cli.cmdoptions import ( + _convert_python_version, make_search_scope, +) + + +@pytest.mark.parametrize( + 'no_index, suppress_no_index, expected_index_urls', [ + (False, False, ['default_url', 'url1', 'url2']), + (False, True, ['default_url', 'url1', 'url2']), + (True, False, []), + # Passing suppress_no_index=True suppresses no_index=True. + (True, True, ['default_url', 'url1', 'url2']), + ], +) +def test_make_search_scope(no_index, suppress_no_index, expected_index_urls): + """ + :param expected: the expected index_urls value. + """ + options = pretend.stub( + find_links=['link1'], + index_url='default_url', + extra_index_urls=['url1', 'url2'], + no_index=no_index, + ) + search_scope = make_search_scope( + options, suppress_no_index=suppress_no_index, + ) + assert search_scope.find_links == ['link1'] + assert search_scope.index_urls == expected_index_urls @pytest.mark.parametrize('value, expected', [ diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index 4ebcaf1b735..497130dc30c 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -11,6 +11,7 @@ _check_link_requires_python, _clean_link, _determine_base_url, _egg_info_matches, _find_name_version_sep, _get_html_page, ) +from pip._internal.models.search_scope import SearchScope from pip._internal.models.target_python import TargetPython from tests.lib import CURRENT_PY_VERSION_INFO, make_test_finder @@ -154,9 +155,12 @@ def test_create__target_python(self): """ Test that target_python is passed to CandidateEvaluator as is. """ + search_scope = SearchScope([], []) target_python = TargetPython(py_version_info=(3, 7, 3)) finder = PackageFinder.create( - [], [], session=object(), target_python=target_python, + search_scope=search_scope, + session=object(), + target_python=target_python, ) evaluator = finder.candidate_evaluator actual_target_python = evaluator._target_python @@ -237,8 +241,11 @@ def test_iter_secure_origins__none_trusted_hosts(self): """ # Use PackageFinder.create() rather than make_test_finder() # to make sure we're really passing trusted_hosts=None. + search_scope = SearchScope([], []) finder = PackageFinder.create( - [], [], trusted_hosts=None, session=object(), + search_scope=search_scope, + trusted_hosts=None, + session=object(), ) actual = list(finder.iter_secure_origins()) diff --git a/tests/unit/test_unit_outdated.py b/tests/unit/test_unit_outdated.py index e337040eb7a..5a8eb5c1e9f 100644 --- a/tests/unit/test_unit_outdated.py +++ b/tests/unit/test_unit_outdated.py @@ -58,8 +58,8 @@ def get_metadata_lines(self, name): def _options(): ''' Some default options that we pass to outdated.pip_version_check ''' return pretend.stub( - find_links=False, extra_index_urls=[], index_url='default_url', - pre=False, trusted_hosts=False, cache_dir='', + find_links=False, index_url='default_url', extra_index_urls=[], + no_index=False, pre=False, trusted_hosts=False, cache_dir='', )