From f59347ed6369977c87d169c5b277f8882f4ba140 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 11 Sep 2023 18:14:56 -0400 Subject: [PATCH 1/4] Expand the pipenv resolver to be more customizeable for what python version is specified and how markers should evaluate. --- Pipfile | 8 +- Pipfile.lock | 46 +++-- pipenv/environment.py | 9 +- .../patched/pip/_vendor/packaging/markers.py | 33 +++- pipenv/project.py | 23 +-- pipenv/resolver.py | 109 ------------ pipenv/routines/install.py | 4 - pipenv/utils/dependencies.py | 10 +- pipenv/utils/indexes.py | 55 +----- pipenv/utils/project.py | 21 --- pipenv/utils/requirementslib.py | 161 +----------------- pipenv/utils/resolver.py | 9 +- .../patches/patched/packaging_resolver.patch | 74 ++++++++ 13 files changed, 160 insertions(+), 402 deletions(-) create mode 100644 tasks/vendoring/patches/patched/packaging_resolver.patch diff --git a/Pipfile b/Pipfile index 8c1b89f573..4e3e32ebd4 100644 --- a/Pipfile +++ b/Pipfile @@ -11,7 +11,7 @@ sphinxcontrib-spelling = "==7.*" click = "==8.0.3" pypiserver = "==1.*" stdeb = {version="*", sys_platform = "== 'linux'"} -zipp = {version = "==3.6.0", markers = "python_version < '3.10'"} +zipp = {version = "==3.6.0"} pre-commit = "==2.*" atomicwrites = {version = "*", sys_platform = "== 'win32'"} pytest-cov = "==3.*" @@ -41,3 +41,9 @@ test = "pytest -vvs" [pipenv] allow_prereleases = true + +[resolver] +python_full_version = ">=3.7.0,<3.12.0" +python_version = ">=3.7,<3.12" +python = "3.7" +os_name="linux,nt,darwin" diff --git a/Pipfile.lock b/Pipfile.lock index b5b4957137..6c2beb911d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8a84150804ee1b1aa65d180190c3795476aea5d3a49b530663ce52a03b5ca8c1" + "sha256": "5fb0ca651d5f094561439576419ea30bfb0e8ae41c3b3b4840799a3606a697a8" }, "pipfile-spec": 6, "requires": {}, @@ -16,11 +16,11 @@ "default": { "pytz": { "hashes": [ - "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", - "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" + "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", + "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7" ], "index": "pypi", - "version": "==2023.3" + "version": "==2023.3.post1" } }, "develop": { @@ -108,12 +108,12 @@ }, "build": { "hashes": [ - "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171", - "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269" + "sha256:538aab1b64f9828977f84bc63ae570b060a8ed1be419e7870b8b4fc5e6ea553b", + "sha256:589bf99a67df7c9cf07ec0ac0e5e2ea5d4b37ac63301c4986d1acb126aa83f8f" ], "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==0.10.0" + "version": "==1.0.3" }, "certifi": { "hashes": [ @@ -476,8 +476,11 @@ "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c", "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb", + "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939", "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", @@ -485,6 +488,7 @@ "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd", "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", @@ -493,6 +497,7 @@ "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007", "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", @@ -500,9 +505,12 @@ "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1", "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c", "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823", "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", @@ -521,7 +529,9 @@ "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", - "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" + "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2", + "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11" ], "markers": "python_version >= '3.7'", "version": "==2.1.3" @@ -725,11 +735,11 @@ }, "pytest": { "hashes": [ - "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32", - "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a" + "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002", + "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069" ], "markers": "python_version >= '3.7'", - "version": "==7.4.0" + "version": "==7.4.2" }, "pytest-cov": { "hashes": [ @@ -756,11 +766,11 @@ }, "pytz": { "hashes": [ - "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", - "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" + "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", + "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7" ], "index": "pypi", - "version": "==2023.3" + "version": "==2023.3.post1" }, "pywin32-ctypes": { "hashes": [ @@ -1078,11 +1088,11 @@ }, "virtualenv": { "hashes": [ - "sha256:29c70bb9b88510f6414ac3e55c8b413a1f96239b6b789ca123437d5e892190cb", - "sha256:772b05bfda7ed3b8ecd16021ca9716273ad9f4467c801f27e83ac73430246dca" + "sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b", + "sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752" ], "markers": "python_version >= '3.7'", - "version": "==20.24.4" + "version": "==20.24.5" }, "waitress": { "hashes": [ @@ -1104,7 +1114,7 @@ "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832", "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc" ], - "markers": "python_version < '3.10'", + "markers": "python_version >= '3.6'", "version": "==3.6.0" } } diff --git a/pipenv/environment.py b/pipenv/environment.py index 26b42e66d2..75162eb35d 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -603,8 +603,15 @@ def get_finder(self, pre: bool = False) -> ContextManager[PackageFinder]: pip_options.cache_dir = self.project.s.PIPENV_CACHE_DIR pip_options.pre = self.pipfile.get("pre", pre) session = pip_command._build_session(pip_options) + python_version = None + requires = self.project.parsed_pipfile.get("resolver", {}) + if "python" in requires: + python_version = requires["python"] finder = get_package_finder( - install_cmd=pip_command, options=pip_options, session=session + install_cmd=pip_command, + options=pip_options, + session=session, + python_versions=python_version, ) yield finder diff --git a/pipenv/patched/pip/_vendor/packaging/markers.py b/pipenv/patched/pip/_vendor/packaging/markers.py index bc16ff4acd..cbf34469fa 100644 --- a/pipenv/patched/pip/_vendor/packaging/markers.py +++ b/pipenv/patched/pip/_vendor/packaging/markers.py @@ -6,8 +6,11 @@ import os import platform import sys +from functools import lru_cache from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from pipenv.patched.pip._vendor.packaging.version import Version, parse +from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet from pipenv.patched.pip._vendor.pyparsing import ( # noqa: N817 Forward, Group, @@ -231,13 +234,26 @@ def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: lhs, op, rhs = marker if isinstance(lhs, Variable): - lhs_value = _get_env(environment, lhs.value) + lookup_key = lhs.value + lhs_value = _get_env(environment, lookup_key) rhs_value = rhs.value else: lhs_value = lhs.value - rhs_value = _get_env(environment, rhs.value) + lookup_key = rhs.value + rhs_value = _get_env(environment, lookup_key) + + if lookup_key in ["python_version", "python_full_version"] and lhs_value and (',' in lhs_value or '<' in lhs_value or '>' in lhs_value): + lhs_spec_set = SpecifierSet(lhs_value) + rhs_version = parse(rhs_value) + is_satisfied = lhs_spec_set.contains(rhs_version) + elif lookup_key == "os_name" and lhs_value and lhs_value == 'os_name' and ',' in lhs_value: + # Handle multiple os_name values + os_names = lhs_value.split(',') + is_satisfied = any(_eval_op(rhs_value, op, os_name) for os_name in os_names) + else: + is_satisfied = _eval_op(lhs_value, op, rhs_value) - groups[-1].append(_eval_op(lhs_value, op, rhs_value)) + groups[-1].append(is_satisfied) else: assert marker in ["and", "or"] if marker == "or": @@ -254,10 +270,12 @@ def format_full_version(info: "sys._version_info") -> str: return version +@lru_cache() def default_environment() -> Dict[str, str]: + from pipenv.project import Project iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name - return { + defaults = { "implementation_name": implementation_name, "implementation_version": iver, "os_name": os.name, @@ -270,6 +288,13 @@ def default_environment() -> Dict[str, str]: "python_version": ".".join(platform.python_version_tuple()[:2]), "sys_platform": sys.platform, } + project = Project() + requires = project.parsed_pipfile.get("resolver", {}) + for k in defaults: + if requires.get(k): + defaults[k] = requires[k] + + return defaults class Marker: diff --git a/pipenv/project.py b/pipenv/project.py index b96487c108..71e3dbb93e 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -56,7 +56,6 @@ proper_case, ) from pipenv.utils.locking import atomic_open_for_write -from pipenv.utils.project import get_default_pyproject_backend from pipenv.utils.requirements import normalize_name from pipenv.utils.shell import ( find_requirements, @@ -93,6 +92,7 @@ NON_CATEGORY_SECTIONS = { "pipenv", "requires", + "resolver", "scripts", "source", } @@ -707,27 +707,6 @@ def _parse_pipfile( # Fallback to toml parser, for large files. return toml.loads(contents) - def _read_pyproject(self) -> None: - pyproject = self.path_to("pyproject.toml") - if os.path.exists(pyproject): - self._pyproject = toml.load(pyproject) - build_system = self._pyproject.get("build-system", None) - if not os.path.exists(self.path_to("setup.py")): - if not build_system or not build_system.get("requires"): - build_system = { - "requires": ["setuptools>=40.8.0", "wheel"], - "build-backend": get_default_pyproject_backend(), - } - self._build_system = build_system - - @property - def build_requires(self) -> list[str]: - return self._build_system.get("requires", ["setuptools>=40.8.0", "wheel"]) - - @property - def build_backend(self) -> str: - return self._build_system.get("build-backend", get_default_pyproject_backend()) - @property def settings(self) -> tomlkit.items.Table | dict[str, str | bool]: """A dictionary of the settings added to the Pipfile.""" diff --git a/pipenv/resolver.py b/pipenv/resolver.py index ef4e11b302..837dc59681 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -71,10 +71,6 @@ def get_parser(): return parser -def which(*args, **kwargs): - return sys.executable - - def handle_parsed_args(parsed): if parsed.verbose: os.environ["PIPENV_VERBOSITY"] = "1" @@ -344,32 +340,6 @@ def strip_version(specifier): specifier = specifier[len(op) :] return specifier - @property - def parent_deps(self): - if not self._parent_deps: - self._parent_deps = self.get_parent_deps(unnest=False) - return self._parent_deps - - @property - def flattened_parents(self): - if not self._flattened_parents: - self._flattened_parents = self.get_parent_deps(unnest=True) - return self._flattened_parents - - @property - def parents_in_pipfile(self): - if not self._parents_in_pipfile: - self._parents_in_pipfile = [ - p - for p in self.flattened_parents - if p.normalized_name in self.pipfile_packages - ] - return self._parents_in_pipfile - - @property - def is_updated(self): - return self.entry.specifiers != self.lockfile_entry.specifiers - @property def requirements(self): if not self._requires: @@ -387,60 +357,6 @@ def updated_version(self): def updated_specifier(self) -> str: return str(self.entry.specifier) - @property - def original_specifier(self) -> str: - return self.lockfile_entry.specifiers - - @property - def original_version(self): - if self.original_specifier: - return self.strip_version(self.original_specifier) - return None - - def validate_specifiers(self): - if self.is_in_pipfile and not self.pipfile_entry.editable: - return self.pipfile_entry.requirement.specifier.contains(self.updated_version) - return True - - def get_dependency(self, name): - if self.requirements: - return next( - iter( - dep - for dep in self.requirements.get("dependencies", []) - if dep and dep.get("package_name", "") == name - ), - {}, - ) - return {} - - def get_parent_deps(self, unnest=False): - from pipenv.patched.pip._vendor.packaging.specifiers import Specifier - - parents = [] - for spec in self.reverse_deps.get(self.normalized_name, {}).get("parents", set()): - spec_match = next(iter(c for c in Specifier._operators if c in spec), None) - name = spec - parent = None - if spec_match is not None: - spec_index = spec.index(spec_match) - specifier = self.clean_specifier( - spec[spec_index : len(spec_match)] - ).strip() - name_start = spec_index + len(spec_match) - name = spec[name_start:].strip() - parent = self.create_parent(name, specifier) - else: - name = spec - parent = self.create_parent(name) - if parent is not None: - parents.append(parent) - if not unnest or parent.pipfile_name is not None: - continue - if self.reverse_deps.get(parent.normalized_name, {}).get("parents", set()): - parents.extend(parent.flattened_parents) - return parents - def get_constraints(self): """ Retrieve all of the relevant constraints, aggregated from the pipfile, resolver, @@ -451,16 +367,6 @@ def get_constraints(self): """ return self.resolver.parsed_constraints - def get_pipfile_constraint(self): - """ - Retrieve the version constraint from the pipfile if it is specified there, - otherwise check the constraints of the parent dependencies and their conflicts. - - :return: An **InstallRequirement** instance representing a version constraint - """ - if self.is_in_pipfile: - return self.pipfile_entry - def validate_constraints(self): """ Retrieves the full set of available constraints and iterate over them, validating @@ -490,20 +396,6 @@ def validate_constraints(self): raise DependencyConflict(msg) return True - def check_flattened_parents(self): - for parent in self.parents_in_pipfile: - if not parent.updated_specifier: - continue - if not parent.validate_specifiers(): - from pipenv.exceptions import DependencyConflict - - msg = ( - f"Cannot resolve conflicting versions: (Root: {self.name}) " - f"{parent.pipfile_name}{parent.pipfile_entry.requirement.specifiers} (Pipfile) " - f"Incompatible with {parent.name}{parent.updated_specifiers} (resolved)\n" - ) - raise DependencyConflict(msg) - def __getattribute__(self, key): result = None old_version = ["was_", "had_", "old_"] @@ -591,7 +483,6 @@ def resolve( ): return resolve_deps( packages, - which, project=project, pre=pre, category=category, diff --git a/pipenv/routines/install.py b/pipenv/routines/install.py index 2f135b4354..59fba08d71 100644 --- a/pipenv/routines/install.py +++ b/pipenv/routines/install.py @@ -17,7 +17,6 @@ from pipenv.utils.indexes import get_source_list from pipenv.utils.internet import download_file, is_valid_url from pipenv.utils.pip import ( - get_trusted_hosts, pip_install_deps, ) from pipenv.utils.pipfile import ensure_pipfile @@ -508,9 +507,6 @@ def batch_install( search_all_sources = project.settings.get("install_search_all_sources", False) sources = get_source_list( project, - index=None, - extra_indexes=None, - trusted_hosts=get_trusted_hosts(), pypi_mirror=pypi_mirror, ) if search_all_sources: diff --git a/pipenv/utils/dependencies.py b/pipenv/utils/dependencies.py index 90475c7689..eae7f387c3 100644 --- a/pipenv/utils/dependencies.py +++ b/pipenv/utils/dependencies.py @@ -343,12 +343,6 @@ def is_star(val): return isinstance(val, str) and val == "*" -def is_pinned(val): - if isinstance(val, Mapping): - val = val.get("version") - return isinstance(val, str) and val.startswith("==") - - def is_pinned_requirement(ireq): """ Returns whether an InstallRequirement is a "pinned" requirement. @@ -1060,9 +1054,9 @@ def install_req_from_pipfile(name, pipfile): vcs_url = vcs_url_parts[0] fallback_ref = vcs_url_parts[1] req_str = f"{vcs_url}@{_pipfile.get('ref', fallback_ref)}{extras_str}" - if not req_str.startswith(f"{vcs}+"): + if not req_str.lstrip().startswith(f"{vcs}+"): req_str = f"{vcs}+{req_str}" - if f"{vcs}+file://" in req_str or _pipfile.get("editable", False): + if _pipfile.get("editable", False) or f"{vcs}+file://" in req_str: req_str = ( f"-e {req_str}#egg={name}{extras_str}{subdirectory.replace('#', '&')}" ) diff --git a/pipenv/utils/indexes.py b/pipenv/utils/indexes.py index 99485caf5e..48958dc0f4 100644 --- a/pipenv/utils/indexes.py +++ b/pipenv/utils/indexes.py @@ -1,18 +1,11 @@ from __future__ import annotations import re -from collections.abc import Mapping +from argparse import ArgumentParser from pipenv.exceptions import PipenvUsageError from pipenv.patched.pip._vendor.urllib3.util import parse_url -from pipenv.utils.constants import MYPY_RUNNING - -from .internet import create_mirror_source, is_pypi_url - -if MYPY_RUNNING: - from typing import List, Optional, Union # noqa - - from pipenv.project import Project, TSource # noqa +from pipenv.utils.internet import create_mirror_source, is_pypi_url def prepare_pip_source_args(sources, pip_args=None): @@ -44,49 +37,11 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args -def get_project_index( - project: Project, - index: str | TSource | None = None, - trusted_hosts: list[str] | None = None, -) -> TSource: - from pipenv.project import SourceNotFound - - if trusted_hosts is None: - trusted_hosts = [] - if isinstance(index, Mapping): - return project.find_source(index.get("url")) - try: - source = project.find_source(index) - except SourceNotFound: - index_url = parse_url(index) - src_name = project.src_name_from_url(index) - verify_ssl = index_url.host not in trusted_hosts - source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} - return source - - def get_source_list( - project: Project, - index: str | TSource | None = None, - extra_indexes: str | list[str] | None = None, - trusted_hosts: list[str] | None = None, + project, pypi_mirror: str | None = None, -) -> list[TSource]: +) -> list[dict[str, str | bool]]: sources = project.sources[:] - if index: - sources.append(get_project_index(project, index)) - if extra_indexes: - if isinstance(extra_indexes, str): - extra_indexes = [extra_indexes] - - for source in extra_indexes: - extra_src = get_project_index(project, source) - if not sources or extra_src["url"] != sources[0]["url"]: - sources.append(extra_src) - - for source in project.sources: - if not sources or source["url"] != sources[0]["url"]: - sources.append(source) if pypi_mirror: sources = [ @@ -99,8 +54,6 @@ def get_source_list( def parse_indexes(line, strict=False): - from argparse import ArgumentParser - comment_re = re.compile(r"(?:^|\s+)#.*$") line = comment_re.sub("", line) parser = ArgumentParser("indexes", allow_abbrev=False) diff --git a/pipenv/utils/project.py b/pipenv/utils/project.py index 84166961bc..44fa1c95b8 100644 --- a/pipenv/utils/project.py +++ b/pipenv/utils/project.py @@ -1,9 +1,6 @@ import os -from functools import lru_cache from pipenv import exceptions -from pipenv.patched.pip._vendor.packaging.version import parse as parse_version -from pipenv.patched.pip._vendor.pkg_resources import Requirement, get_distribution from pipenv.utils.dependencies import python_version from pipenv.utils.pipfile import ensure_pipfile from pipenv.utils.shell import shorten_path @@ -85,21 +82,3 @@ def ensure_project( categories=categories, ) os.environ["PIP_PYTHON_PATH"] = project.python(system=system) - - -@lru_cache() -def get_setuptools_version(): - # type: () -> Optional[STRING_TYPE] - - setuptools_dist = get_distribution(Requirement("setuptools")) - return getattr(setuptools_dist, "version", None) - - -def get_default_pyproject_backend(): - # type: () -> STRING_TYPE - st_version = get_setuptools_version() - if st_version is not None: - parsed_st_version = parse_version(st_version) - if parsed_st_version >= parse_version("40.8.0"): - return "setuptools.build_meta:__legacy__" - return "setuptools.build_meta" diff --git a/pipenv/utils/requirementslib.py b/pipenv/utils/requirementslib.py index ef78f4b599..ecd4162108 100644 --- a/pipenv/utils/requirementslib.py +++ b/pipenv/utils/requirementslib.py @@ -1,12 +1,9 @@ -import os from collections.abc import ItemsView, Mapping, Sequence, Set -from pathlib import Path from typing import Dict, List, Optional, Tuple, TypeVar, Union from urllib.parse import urlparse, urlsplit, urlunparse from pipenv.patched.pip._internal.commands.install import InstallCommand from pipenv.patched.pip._internal.models.link import Link -from pipenv.patched.pip._internal.models.target_python import TargetPython from pipenv.patched.pip._internal.network.download import Downloader from pipenv.patched.pip._internal.operations.prepare import ( File, @@ -14,13 +11,10 @@ get_file_url, unpack_vcs_link, ) -from pipenv.patched.pip._internal.utils.filetypes import is_archive_file from pipenv.patched.pip._internal.utils.hashes import Hashes -from pipenv.patched.pip._internal.utils.misc import is_installable_dir from pipenv.patched.pip._internal.utils.temp_dir import TempDirectory from pipenv.patched.pip._internal.utils.unpacking import unpack_file -from pipenv.patched.pip._vendor.packaging import specifiers -from pipenv.utils.fileutils import is_valid_url, normalize_path, url_to_path +from pipenv.utils.fileutils import is_valid_url from pipenv.vendor import tomlkit STRING_TYPE = Union[bytes, str, str] @@ -60,20 +54,6 @@ ] -def strip_ssh_from_git_uri(uri): - # type: (S) -> S - """Return git+ssh:// formatted URI to git+git@ format.""" - if isinstance(uri, str) and "git+ssh://" in uri: - parsed = urlparse(uri) - # split the path on the first separating / so we can put the first segment - # into the 'netloc' section with a : separator - path_part, _, path = parsed.path.lstrip("/").partition("/") - path = f"/{path}" - parsed = parsed._replace(netloc=f"{parsed.netloc}:{path_part}", path=path) - uri = urlunparse(parsed).replace("git+ssh://", "git+", 1) - return uri - - def add_ssh_scheme_to_git_uri(uri): # type: (S) -> S """Cleans VCS uris from pip format.""" @@ -113,116 +93,6 @@ def is_editable(pipfile_entry): return False -def is_star(val): - # type: (PipfileType) -> bool - return (isinstance(val, str) and val == "*") or ( - isinstance(val, Mapping) and val.get("version", "") == "*" - ) - - -def convert_entry_to_path(path): - # type: (Dict[S, Union[S, bool, Tuple[S], List[S]]]) -> S - """Convert a pipfile entry to a string.""" - - if not isinstance(path, Mapping): - raise TypeError(f"expecting a mapping, received {path!r}") - - if not any(key in path for key in ["file", "path"]): - raise ValueError(f"missing path-like entry in supplied mapping {path!r}") - - if "file" in path: - path = url_to_path(path["file"]) - - elif "path" in path: - path = path["path"] - return Path(os.fsdecode(path)).as_posix() if os.name == "nt" else os.fsdecode(path) - - -def is_installable_file(path): - # type: (PipfileType) -> bool - """Determine if a path can potentially be installed.""" - - if isinstance(path, Mapping): - path = convert_entry_to_path(path) - - # If the string starts with a valid specifier operator, test if it is a valid - # specifier set before making a path object (to avoid breaking windows) - if any(path.startswith(spec) for spec in "!=<>~"): - try: - specifiers.SpecifierSet(path) - # If this is not a valid specifier, just move on and try it as a path - except specifiers.InvalidSpecifier: - pass - else: - return False - - parsed = urlparse(path) - is_local = ( - not parsed.scheme - or parsed.scheme == "file" - or (len(parsed.scheme) == 1 and os.name == "nt") - ) - if parsed.scheme and parsed.scheme == "file": - path = os.fsdecode(url_to_path(path)) - normalized_path = normalize_path(path) - if is_local and not os.path.exists(normalized_path): - return False - - is_archive = is_archive_file(normalized_path) - is_local_project = os.path.isdir(normalized_path) and is_installable_dir( - normalized_path - ) - if is_local and is_local_project or is_archive: - return True - - if not is_local and is_archive_file(parsed.path): - return True - - return False - - -def get_dist_metadata(dist): - from email.parser import FeedParser - - from pipenv.patched.pip._vendor.pkg_resources import DistInfoDistribution - - if isinstance(dist, DistInfoDistribution) and dist.has_metadata("METADATA"): - metadata = dist.get_metadata("METADATA") - elif dist.has_metadata("PKG-INFO"): - metadata = dist.get_metadata("PKG-INFO") - else: - metadata = "" - - feed_parser = FeedParser() - feed_parser.feed(metadata) - return feed_parser.close() - - -def get_setup_paths(base_path, subdirectory=None): - # type: (S, Optional[S]) -> Dict[S, Optional[S]] - if base_path is None: - raise TypeError("must provide a path to derive setup paths from") - setup_py = os.path.join(base_path, "setup.py") - setup_cfg = os.path.join(base_path, "setup.cfg") - pyproject_toml = os.path.join(base_path, "pyproject.toml") - if subdirectory is not None: - base_path = os.path.join(base_path, subdirectory) - subdir_setup_py = os.path.join(subdirectory, "setup.py") - subdir_setup_cfg = os.path.join(subdirectory, "setup.cfg") - subdir_pyproject_toml = os.path.join(subdirectory, "pyproject.toml") - if subdirectory and os.path.exists(subdir_setup_py): - setup_py = subdir_setup_py - if subdirectory and os.path.exists(subdir_setup_cfg): - setup_cfg = subdir_setup_cfg - if subdirectory and os.path.exists(subdir_pyproject_toml): - pyproject_toml = subdir_pyproject_toml - return { - "setup_py": setup_py if os.path.exists(setup_py) else None, - "setup_cfg": setup_cfg if os.path.exists(setup_cfg) else None, - "pyproject_toml": pyproject_toml if os.path.exists(pyproject_toml) else None, - } - - def prepare_pip_source_args(sources, pip_args=None): # type: (List[Dict[S, Union[S, bool]]], Optional[List[S]]) -> List[S] if pip_args is None: @@ -247,35 +117,6 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args -def get_package_finder( - install_cmd=None, - options=None, - session=None, - platform=None, - python_versions=None, - abi=None, - implementation=None, - ignore_requires_python=None, -): - """Reduced Shim for compatibility to generate package finders.""" - py_version_info = None - if python_versions: - py_version_info_python = max(python_versions) - py_version_info = tuple([int(part) for part in py_version_info_python]) - target_python = TargetPython( - platforms=[platform] if platform else None, - py_version_info=py_version_info, - abis=[abi] if abi else None, - implementation=implementation, - ) - return install_cmd._build_package_finder( - options=options, - session=session, - target_python=target_python, - ignore_requires_python=ignore_requires_python, - ) - - _UNSET = object() _REMAP_EXIT = object() diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 06c8c8cb4a..76fbb94d34 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -69,8 +69,7 @@ def get_package_finder( """Reduced Shim for compatibility to generate package finders.""" py_version_info = None if python_versions: - py_version_info_python = max(python_versions) - py_version_info = tuple([int(part) for part in py_version_info_python]) + py_version_info = [int(part) for part in python_versions.split(".")] target_python = TargetPython( platforms=[platform] if platform else None, py_version_info=py_version_info, @@ -325,10 +324,15 @@ def prepare_index_lookup(self): @cached_property def package_finder(self): + python_version = None + requires = self.project.parsed_pipfile.get("resolver", {}) + if "python" in requires: + python_version = requires["python"] finder = get_package_finder( install_cmd=self.pip_command, options=self.pip_options, session=self.session, + python_versions=python_version, ) return finder @@ -863,7 +867,6 @@ def venv_resolve_deps( def resolve_deps( deps, - which, project, sources=None, python=False, diff --git a/tasks/vendoring/patches/patched/packaging_resolver.patch b/tasks/vendoring/patches/patched/packaging_resolver.patch new file mode 100644 index 0000000000..dfffe0ed87 --- /dev/null +++ b/tasks/vendoring/patches/patched/packaging_resolver.patch @@ -0,0 +1,74 @@ +diff --git a/pipenv/patched/pip/_vendor/packaging/markers.py b/pipenv/patched/pip/_vendor/packaging/markers.py +index bc16ff4a..cbf34469 100644 +--- a/pipenv/patched/pip/_vendor/packaging/markers.py ++++ b/pipenv/patched/pip/_vendor/packaging/markers.py +@@ -6,8 +6,11 @@ import operator + import os + import platform + import sys ++from functools import lru_cache + from typing import Any, Callable, Dict, List, Optional, Tuple, Union + ++from pipenv.patched.pip._vendor.packaging.version import Version, parse ++from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet + from pipenv.patched.pip._vendor.pyparsing import ( # noqa: N817 + Forward, + Group, +@@ -231,13 +234,26 @@ def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: + lhs, op, rhs = marker + + if isinstance(lhs, Variable): +- lhs_value = _get_env(environment, lhs.value) ++ lookup_key = lhs.value ++ lhs_value = _get_env(environment, lookup_key) + rhs_value = rhs.value + else: + lhs_value = lhs.value +- rhs_value = _get_env(environment, rhs.value) ++ lookup_key = rhs.value ++ rhs_value = _get_env(environment, lookup_key) ++ ++ if lookup_key in ["python_version", "python_full_version"] and lhs_value and (',' in lhs_value or '<' in lhs_value or '>' in lhs_value): ++ lhs_spec_set = SpecifierSet(lhs_value) ++ rhs_version = parse(rhs_value) ++ is_satisfied = lhs_spec_set.contains(rhs_version) ++ elif lookup_key == "os_name" and lhs_value and lhs_value == 'os_name' and ',' in lhs_value: ++ # Handle multiple os_name values ++ os_names = lhs_value.split(',') ++ is_satisfied = any(_eval_op(rhs_value, op, os_name) for os_name in os_names) ++ else: ++ is_satisfied = _eval_op(lhs_value, op, rhs_value) + +- groups[-1].append(_eval_op(lhs_value, op, rhs_value)) ++ groups[-1].append(is_satisfied) + else: + assert marker in ["and", "or"] + if marker == "or": +@@ -254,10 +270,12 @@ def format_full_version(info: "sys._version_info") -> str: + return version + + ++@lru_cache() + def default_environment() -> Dict[str, str]: ++ from pipenv.project import Project + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name +- return { ++ defaults = { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, +@@ -270,6 +288,13 @@ def default_environment() -> Dict[str, str]: + "python_version": ".".join(platform.python_version_tuple()[:2]), + "sys_platform": sys.platform, + } ++ project = Project() ++ requires = project.parsed_pipfile.get("resolver", {}) ++ for k in defaults: ++ if requires.get(k): ++ defaults[k] = requires[k] ++ ++ return defaults + + + class Marker: From c0cccd815f074f5d523ed9d656153a399e9e643e Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Mon, 11 Sep 2023 19:24:34 -0400 Subject: [PATCH 2/4] Only use resolver defaults in the resolver --- .../patched/pip/_vendor/packaging/markers.py | 20 +++++++++---------- pipenv/utils/dependencies.py | 4 +++- pipenv/utils/resolver.py | 4 +++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pipenv/patched/pip/_vendor/packaging/markers.py b/pipenv/patched/pip/_vendor/packaging/markers.py index cbf34469fa..95f73dbaa1 100644 --- a/pipenv/patched/pip/_vendor/packaging/markers.py +++ b/pipenv/patched/pip/_vendor/packaging/markers.py @@ -6,10 +6,8 @@ import os import platform import sys -from functools import lru_cache from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from pipenv.patched.pip._vendor.packaging.version import Version, parse from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet from pipenv.patched.pip._vendor.pyparsing import ( # noqa: N817 Forward, @@ -244,8 +242,8 @@ def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: if lookup_key in ["python_version", "python_full_version"] and lhs_value and (',' in lhs_value or '<' in lhs_value or '>' in lhs_value): lhs_spec_set = SpecifierSet(lhs_value) - rhs_version = parse(rhs_value) - is_satisfied = lhs_spec_set.contains(rhs_version) + rhs_spec_set = SpecifierSet(rhs_value) + is_satisfied = any(lhs_spec_set.contains(ver) for ver in rhs_spec_set) elif lookup_key == "os_name" and lhs_value and lhs_value == 'os_name' and ',' in lhs_value: # Handle multiple os_name values os_names = lhs_value.split(',') @@ -270,8 +268,7 @@ def format_full_version(info: "sys._version_info") -> str: return version -@lru_cache() -def default_environment() -> Dict[str, str]: +def default_environment(resolve_phase=False) -> Dict[str, str]: from pipenv.project import Project iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name @@ -288,11 +285,12 @@ def default_environment() -> Dict[str, str]: "python_version": ".".join(platform.python_version_tuple()[:2]), "sys_platform": sys.platform, } - project = Project() - requires = project.parsed_pipfile.get("resolver", {}) - for k in defaults: - if requires.get(k): - defaults[k] = requires[k] + if resolve_phase: + project = Project() + requires = project.parsed_pipfile.get("resolver", {}) + for k in defaults: + if requires.get(k): + defaults[k] = requires[k] return defaults diff --git a/pipenv/utils/dependencies.py b/pipenv/utils/dependencies.py index eae7f387c3..1f34e4e2ee 100644 --- a/pipenv/utils/dependencies.py +++ b/pipenv/utils/dependencies.py @@ -147,7 +147,9 @@ def pep423_name(name): def translate_markers(pipfile_entry): from pipenv.patched.pip._vendor.packaging.markers import default_environment - allowed_marker_keys = ["markers"] + list(default_environment().keys()) + allowed_marker_keys = ["markers"] + list( + default_environment(resolve_phase=True).keys() + ) provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else [] pipfile_markers = set(provided_keys) & set(allowed_marker_keys) new_pipfile = dict(pipfile_entry).copy() diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 76fbb94d34..025fb087f1 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -25,6 +25,7 @@ from pipenv.patched.pip._internal.req.req_install import InstallRequirement from pipenv.patched.pip._internal.utils.temp_dir import global_tempdir_manager from pipenv.patched.pip._vendor import pkg_resources, rich +from pipenv.patched.pip._vendor.packaging.markers import default_environment from pipenv.project import Project from pipenv.utils.fileutils import create_tracked_tempdir from pipenv.utils.requirements import normalize_name @@ -161,7 +162,8 @@ def check_if_package_req_skipped( self, req: InstallRequirement, ) -> bool: - if req.markers and not req.markers.evaluate(): + defaults = default_environment(resolve_phase=True) + if req.markers and not req.markers.evaluate(defaults): err.print( f"Could not find a matching version of {req}; {req.markers} for your environment, " "its dependencies will be skipped.", From ae40af4e24e6c34ac8322081627dd8b663b1a511 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 12 Sep 2023 07:39:05 -0400 Subject: [PATCH 3/4] simplify approach (no python ranges for now) and do not include as a patch but rather monkeypatch it for the lifespan of the resolver. --- Pipfile | 10 +-- Pipfile.lock | 4 +- pipenv/environment.py | 6 +- .../patched/pip/_vendor/packaging/markers.py | 34 ++------- pipenv/resolver.py | 34 +++++++++ pipenv/utils/dependencies.py | 4 +- pipenv/utils/resolver.py | 8 +- .../patches/patched/packaging_resolver.patch | 74 ------------------- 8 files changed, 53 insertions(+), 121 deletions(-) delete mode 100644 tasks/vendoring/patches/patched/packaging_resolver.patch diff --git a/Pipfile b/Pipfile index 4e3e32ebd4..c723959f71 100644 --- a/Pipfile +++ b/Pipfile @@ -11,7 +11,7 @@ sphinxcontrib-spelling = "==7.*" click = "==8.0.3" pypiserver = "==1.*" stdeb = {version="*", sys_platform = "== 'linux'"} -zipp = {version = "==3.6.0"} +zipp = {version = "==3.6.0", markers = "python_version < '3.10'"} pre-commit = "==2.*" atomicwrites = {version = "*", sys_platform = "== 'win32'"} pytest-cov = "==3.*" @@ -43,7 +43,7 @@ test = "pytest -vvs" allow_prereleases = true [resolver] -python_full_version = ">=3.7.0,<3.12.0" -python_version = ">=3.7,<3.12" -python = "3.7" -os_name="linux,nt,darwin" +python_full_version = "3.7.13" +python_version = "3.7" +finder_python = "3.7" +os_name = "win32" diff --git a/Pipfile.lock b/Pipfile.lock index 6c2beb911d..77928fbc1c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5fb0ca651d5f094561439576419ea30bfb0e8ae41c3b3b4840799a3606a697a8" + "sha256": "6ff45bb0e1b4ae98996193a95f3e0980fa18604c69f804d52c2173188631969a" }, "pipfile-spec": 6, "requires": {}, @@ -1114,7 +1114,7 @@ "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832", "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc" ], - "markers": "python_version >= '3.6'", + "markers": "python_version < '3.10'", "version": "==3.6.0" } } diff --git a/pipenv/environment.py b/pipenv/environment.py index 75162eb35d..100cfc61e9 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -604,9 +604,9 @@ def get_finder(self, pre: bool = False) -> ContextManager[PackageFinder]: pip_options.pre = self.pipfile.get("pre", pre) session = pip_command._build_session(pip_options) python_version = None - requires = self.project.parsed_pipfile.get("resolver", {}) - if "python" in requires: - python_version = requires["python"] + resolver = self.project.parsed_pipfile.get("resolver", {}) + if "finder_python" in resolver: + python_version = resolver["finder_python"] finder = get_package_finder( install_cmd=pip_command, options=pip_options, diff --git a/pipenv/patched/pip/_vendor/packaging/markers.py b/pipenv/patched/pip/_vendor/packaging/markers.py index 95f73dbaa1..fc0386f6b7 100644 --- a/pipenv/patched/pip/_vendor/packaging/markers.py +++ b/pipenv/patched/pip/_vendor/packaging/markers.py @@ -8,7 +8,6 @@ import sys from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet from pipenv.patched.pip._vendor.pyparsing import ( # noqa: N817 Forward, Group, @@ -232,26 +231,12 @@ def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: lhs, op, rhs = marker if isinstance(lhs, Variable): - lookup_key = lhs.value - lhs_value = _get_env(environment, lookup_key) + lhs_value = _get_env(environment, lhs.value) rhs_value = rhs.value else: lhs_value = lhs.value - lookup_key = rhs.value - rhs_value = _get_env(environment, lookup_key) - - if lookup_key in ["python_version", "python_full_version"] and lhs_value and (',' in lhs_value or '<' in lhs_value or '>' in lhs_value): - lhs_spec_set = SpecifierSet(lhs_value) - rhs_spec_set = SpecifierSet(rhs_value) - is_satisfied = any(lhs_spec_set.contains(ver) for ver in rhs_spec_set) - elif lookup_key == "os_name" and lhs_value and lhs_value == 'os_name' and ',' in lhs_value: - # Handle multiple os_name values - os_names = lhs_value.split(',') - is_satisfied = any(_eval_op(rhs_value, op, os_name) for os_name in os_names) - else: - is_satisfied = _eval_op(lhs_value, op, rhs_value) - - groups[-1].append(is_satisfied) + rhs_value = _get_env(environment, rhs.value) + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) else: assert marker in ["and", "or"] if marker == "or": @@ -268,11 +253,10 @@ def format_full_version(info: "sys._version_info") -> str: return version -def default_environment(resolve_phase=False) -> Dict[str, str]: - from pipenv.project import Project +def default_environment() -> Dict[str, str]: iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name - defaults = { + return { "implementation_name": implementation_name, "implementation_version": iver, "os_name": os.name, @@ -285,14 +269,6 @@ def default_environment(resolve_phase=False) -> Dict[str, str]: "python_version": ".".join(platform.python_version_tuple()[:2]), "sys_platform": sys.platform, } - if resolve_phase: - project = Project() - requires = project.parsed_pipfile.get("resolver", {}) - for k in defaults: - if requires.get(k): - defaults[k] = requires[k] - - return defaults class Marker: diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 837dc59681..f8a198d2a2 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -2,7 +2,9 @@ import json import logging import os +import platform import sys +from typing import Dict try: from functools import cached_property @@ -87,6 +89,34 @@ def handle_parsed_args(parsed): return parsed +def _default_environment_override() -> Dict[str, str]: + from pipenv.patched.pip._vendor.packaging.markers import format_full_version + from pipenv.project import Project + + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + defaults = { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": ".".join(platform.python_version_tuple()[:2]), + "sys_platform": sys.platform, + } + project = Project() + requires = project.parsed_pipfile.get("resolver", {}) + for k in defaults: + if requires.get(k): + defaults[k] = requires[k] + + return defaults + + class Entry: """A resolved entry from a resolver run""" @@ -466,9 +496,12 @@ def resolve_packages( category, constraints=None, ): + from pipenv.patched.pip._vendor.packaging import markers from pipenv.utils.internet import create_mirror_source, replace_pypi_sources from pipenv.utils.resolver import resolve_deps + original_default_environment = markers.default_environment + markers.default_environment = _default_environment_override pypi_mirror_source = ( create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"], "pypi_mirror") if "PIPENV_PYPI_MIRROR" in os.environ @@ -517,6 +550,7 @@ def resolve( json.dump([], fh) else: json.dump(results, fh) + markers.default_environment = original_default_environment if results: return results return [] diff --git a/pipenv/utils/dependencies.py b/pipenv/utils/dependencies.py index 1f34e4e2ee..eae7f387c3 100644 --- a/pipenv/utils/dependencies.py +++ b/pipenv/utils/dependencies.py @@ -147,9 +147,7 @@ def pep423_name(name): def translate_markers(pipfile_entry): from pipenv.patched.pip._vendor.packaging.markers import default_environment - allowed_marker_keys = ["markers"] + list( - default_environment(resolve_phase=True).keys() - ) + allowed_marker_keys = ["markers"] + list(default_environment().keys()) provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else [] pipfile_markers = set(provided_keys) & set(allowed_marker_keys) new_pipfile = dict(pipfile_entry).copy() diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 025fb087f1..da537b2e22 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -25,7 +25,6 @@ from pipenv.patched.pip._internal.req.req_install import InstallRequirement from pipenv.patched.pip._internal.utils.temp_dir import global_tempdir_manager from pipenv.patched.pip._vendor import pkg_resources, rich -from pipenv.patched.pip._vendor.packaging.markers import default_environment from pipenv.project import Project from pipenv.utils.fileutils import create_tracked_tempdir from pipenv.utils.requirements import normalize_name @@ -162,8 +161,7 @@ def check_if_package_req_skipped( self, req: InstallRequirement, ) -> bool: - defaults = default_environment(resolve_phase=True) - if req.markers and not req.markers.evaluate(defaults): + if req.markers and not req.markers.evaluate(): err.print( f"Could not find a matching version of {req}; {req.markers} for your environment, " "its dependencies will be skipped.", @@ -328,8 +326,8 @@ def prepare_index_lookup(self): def package_finder(self): python_version = None requires = self.project.parsed_pipfile.get("resolver", {}) - if "python" in requires: - python_version = requires["python"] + if "finder_python" in requires: + python_version = requires["finder_python"] finder = get_package_finder( install_cmd=self.pip_command, options=self.pip_options, diff --git a/tasks/vendoring/patches/patched/packaging_resolver.patch b/tasks/vendoring/patches/patched/packaging_resolver.patch deleted file mode 100644 index dfffe0ed87..0000000000 --- a/tasks/vendoring/patches/patched/packaging_resolver.patch +++ /dev/null @@ -1,74 +0,0 @@ -diff --git a/pipenv/patched/pip/_vendor/packaging/markers.py b/pipenv/patched/pip/_vendor/packaging/markers.py -index bc16ff4a..cbf34469 100644 ---- a/pipenv/patched/pip/_vendor/packaging/markers.py -+++ b/pipenv/patched/pip/_vendor/packaging/markers.py -@@ -6,8 +6,11 @@ import operator - import os - import platform - import sys -+from functools import lru_cache - from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -+from pipenv.patched.pip._vendor.packaging.version import Version, parse -+from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet - from pipenv.patched.pip._vendor.pyparsing import ( # noqa: N817 - Forward, - Group, -@@ -231,13 +234,26 @@ def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: - lhs, op, rhs = marker - - if isinstance(lhs, Variable): -- lhs_value = _get_env(environment, lhs.value) -+ lookup_key = lhs.value -+ lhs_value = _get_env(environment, lookup_key) - rhs_value = rhs.value - else: - lhs_value = lhs.value -- rhs_value = _get_env(environment, rhs.value) -+ lookup_key = rhs.value -+ rhs_value = _get_env(environment, lookup_key) -+ -+ if lookup_key in ["python_version", "python_full_version"] and lhs_value and (',' in lhs_value or '<' in lhs_value or '>' in lhs_value): -+ lhs_spec_set = SpecifierSet(lhs_value) -+ rhs_version = parse(rhs_value) -+ is_satisfied = lhs_spec_set.contains(rhs_version) -+ elif lookup_key == "os_name" and lhs_value and lhs_value == 'os_name' and ',' in lhs_value: -+ # Handle multiple os_name values -+ os_names = lhs_value.split(',') -+ is_satisfied = any(_eval_op(rhs_value, op, os_name) for os_name in os_names) -+ else: -+ is_satisfied = _eval_op(lhs_value, op, rhs_value) - -- groups[-1].append(_eval_op(lhs_value, op, rhs_value)) -+ groups[-1].append(is_satisfied) - else: - assert marker in ["and", "or"] - if marker == "or": -@@ -254,10 +270,12 @@ def format_full_version(info: "sys._version_info") -> str: - return version - - -+@lru_cache() - def default_environment() -> Dict[str, str]: -+ from pipenv.project import Project - iver = format_full_version(sys.implementation.version) - implementation_name = sys.implementation.name -- return { -+ defaults = { - "implementation_name": implementation_name, - "implementation_version": iver, - "os_name": os.name, -@@ -270,6 +288,13 @@ def default_environment() -> Dict[str, str]: - "python_version": ".".join(platform.python_version_tuple()[:2]), - "sys_platform": sys.platform, - } -+ project = Project() -+ requires = project.parsed_pipfile.get("resolver", {}) -+ for k in defaults: -+ if requires.get(k): -+ defaults[k] = requires[k] -+ -+ return defaults - - - class Marker: From 1d1373f5889016f7dee9ea4fcd540faf2ec0c22a Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Fri, 15 Sep 2023 23:03:07 -0400 Subject: [PATCH 4/4] undo change to vendored lib --- pipenv/patched/pip/_vendor/packaging/markers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipenv/patched/pip/_vendor/packaging/markers.py b/pipenv/patched/pip/_vendor/packaging/markers.py index fc0386f6b7..bc16ff4acd 100644 --- a/pipenv/patched/pip/_vendor/packaging/markers.py +++ b/pipenv/patched/pip/_vendor/packaging/markers.py @@ -236,6 +236,7 @@ def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: else: lhs_value = lhs.value rhs_value = _get_env(environment, rhs.value) + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) else: assert marker in ["and", "or"]