From 4a85f0b7b83840f0b30c8a8cbb6e5276ab946bbc Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Thu, 7 Sep 2023 10:19:07 -0400 Subject: [PATCH] Fallback to default vcs ref and determine package name from the pip line where possible (#5921) * More proactively determine package name from the pip line where possible, fallback to the file scanning logics. --- news/5921.bugfix.rst | 2 + pipenv/project.py | 13 +++++-- pipenv/routines/install.py | 2 +- pipenv/routines/outdated.py | 2 +- pipenv/routines/uninstall.py | 8 ++-- pipenv/routines/update.py | 2 +- pipenv/utils/dependencies.py | 75 +++++++++++++++++++++--------------- pipenv/utils/locking.py | 3 +- pipenv/utils/resolver.py | 7 +++- 9 files changed, 72 insertions(+), 42 deletions(-) create mode 100644 news/5921.bugfix.rst diff --git a/news/5921.bugfix.rst b/news/5921.bugfix.rst new file mode 100644 index 0000000000..e7d5ed1e67 --- /dev/null +++ b/news/5921.bugfix.rst @@ -0,0 +1,2 @@ +Fallback to default vcs ref when no ref is supplied. +More proactively determine package name from the pip line where possible, fallback to the existing file scanning logics when unable to determine name. diff --git a/pipenv/project.py b/pipenv/project.py index b2f353892f..b96487c108 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -1134,11 +1134,17 @@ def remove_packages_from_pipfile(self, packages): self.write_toml(parsed) def generate_package_pipfile_entry(self, package, pip_line, category=None): + """Generate a package entry from pip install line + given the installreq package and the pip line that generated it. + """ # Don't re-capitalize file URLs or VCSs. if not isinstance(package, InstallRequirement): - package = expansive_install_req_from_line(package.strip()) + package, req_name = expansive_install_req_from_line(package.strip()) + else: + _, req_name = expansive_install_req_from_line(pip_line.strip()) - req_name = determine_package_name(package) + if req_name is None: + req_name = determine_package_name(package) path_specifier = determine_path_specifier(package) vcs_specifier = determine_vcs_specifier(package) name = self.get_package_name_in_pipfile(req_name, category=category) @@ -1173,7 +1179,8 @@ def generate_package_pipfile_entry(self, package, pip_line, category=None): else: vcs_part = pip_line vcs_parts = vcs_part.rsplit("@", 1) - entry["ref"] = vcs_parts[1].split("#", 1)[0].strip() + if len(vcs_parts) > 1: + entry["ref"] = vcs_parts[1].split("#", 1)[0].strip() entry[vcs] = vcs_parts[0].strip() # Check and extract subdirectory fragment diff --git a/pipenv/routines/install.py b/pipenv/routines/install.py index cfc7369a27..2f135b4354 100644 --- a/pipenv/routines/install.py +++ b/pipenv/routines/install.py @@ -209,7 +209,7 @@ def do_install( del os.environ["PYTHONHOME"] st.console.print(f"Resolving {pkg_line}...", markup=False) try: - pkg_requirement = expansive_install_req_from_line( + pkg_requirement, _ = expansive_install_req_from_line( pkg_line, expand_env=True ) except ValueError as e: diff --git a/pipenv/routines/outdated.py b/pipenv/routines/outdated.py index ecaeb71675..0c3646d726 100644 --- a/pipenv/routines/outdated.py +++ b/pipenv/routines/outdated.py @@ -29,7 +29,7 @@ def do_outdated(project, pypi_mirror=None, pre=False, clear=False): for name, deps in project.environment.reverse_dependencies().items() } for result in installed_packages: - dep = expansive_install_req_from_line( + dep, _ = expansive_install_req_from_line( str(result.as_requirement()), expand_env=True ) packages.update(as_pipfile(dep)) diff --git a/pipenv/routines/uninstall.py b/pipenv/routines/uninstall.py index ce38028d24..c3975ad0ff 100644 --- a/pipenv/routines/uninstall.py +++ b/pipenv/routines/uninstall.py @@ -42,9 +42,11 @@ def do_uninstall( raise exceptions.PipenvUsageError("No package provided!", ctx=ctx) if not categories: categories = project.get_package_categories(for_lockfile=True) - editable_pkgs = [ - expansive_install_req_from_line(f"-e {p}").name for p in editable_packages if p - ] + editable_pkgs = [] + for p in editable_packages: + if p: + install_req, name = expansive_install_req_from_line(f"-e {p}") + editable_pkgs.append(name) packages += editable_pkgs package_names = {p for p in packages if p} package_map = {canonicalize_name(p): p for p in packages if p} diff --git a/pipenv/routines/update.py b/pipenv/routines/update.py index 5b155c9559..11897a6c20 100644 --- a/pipenv/routines/update.py +++ b/pipenv/routines/update.py @@ -131,7 +131,7 @@ def upgrade( pipfile_category = get_pipfile_category_using_lockfile_section(category) for package in package_args[:]: - install_req = expansive_install_req_from_line(package, expand_env=True) + install_req, _ = expansive_install_req_from_line(package, expand_env=True) if index_name: install_req.index = index_name name, normalized_name, pipfile_entry = project.generate_package_pipfile_entry( diff --git a/pipenv/utils/dependencies.py b/pipenv/utils/dependencies.py index 760e0902e6..ebf89795a5 100644 --- a/pipenv/utils/dependencies.py +++ b/pipenv/utils/dependencies.py @@ -911,7 +911,7 @@ def replace_with_env(match): def expansive_install_req_from_line( - name: str, + pip_line: str, comes_from: Optional[Union[str, InstallRequirement]] = None, *, use_pep517: Optional[bool] = None, @@ -923,30 +923,43 @@ def expansive_install_req_from_line( user_supplied: bool = False, config_settings: Optional[Dict[str, Union[str, List[str]]]] = None, expand_env: bool = False, -) -> InstallRequirement: - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. - - :param line_source: An optional string describing where the line is from, - for logging purposes in case of an error. +) -> (InstallRequirement, str): + """Create an InstallRequirement from a pip-style requirement line. + InstallRequirement is a pip internal construct that represents an installable requirement, + and is used as an intermediary between the pip command and the resolver. + :param pip_line: A pip-style requirement line. + :param comes_from: The path to the requirements file the line was found in. + :param use_pep517: Whether to use PEP 517/518 when installing the + requirement. + :param isolated: Whether to isolate the requirements when installing them. (likely unused) + :param global_options: Extra global options to be used when installing the install req (likely unused) + :param hash_options: Extra hash options to be used when installing the install req (likely unused) + :param constraint: Whether the requirement is a constraint. + :param line_source: The source of the line (e.g. "requirements.txt"). + :param user_supplied: Whether the requirement was directly provided by the user. + :param config_settings: Configuration settings to be used when installing the install req (likely unused) + :param expand_env: Whether to expand environment variables in the line. (definitely used) + :return: A tuple of the InstallRequirement and the name of the package (if determined). """ - name = name.strip("'") - if name.startswith("-e "): # Editable requirements - name = name.split("-e ")[1] - return install_req_from_editable(name, line_source) - if has_name_with_extras(name): - name = name.split(" @ ", 1)[1] + name = None + pip_line = pip_line.strip("'") + for new_req_symbol in ("@ ", " @ "): # Check for new style pip lines + if new_req_symbol in pip_line: + pip_line_parts = pip_line.split(new_req_symbol, 1) + name = pip_line_parts[0] + pip_line = pip_line_parts[1] + if pip_line.startswith("-e "): # Editable requirements + pip_line = pip_line.split("-e ")[1] + return install_req_from_editable(pip_line, line_source), name if expand_env: - name = expand_env_variables(name) + pip_line = expand_env_variables(pip_line) - vcs_part = name - if "@ " in name: # Check for new style vcs lines - vcs_part = name.split("@ ", 1)[1] + vcs_part = pip_line for vcs in VCS_LIST: if vcs_part.startswith(f"{vcs}+"): link = get_link_from_line(vcs_part) - return InstallRequirement( + install_req = InstallRequirement( None, comes_from, link=link, @@ -957,22 +970,23 @@ def expansive_install_req_from_line( constraint=constraint, user_supplied=user_supplied, ) - if urlparse(name).scheme in ("http", "https", "file") or any( - name.endswith(s) for s in INSTALLABLE_EXTENSIONS + return install_req, name + if urlparse(pip_line).scheme in ("http", "https", "file") or any( + pip_line.endswith(s) for s in INSTALLABLE_EXTENSIONS ): - parts = parse_req_from_line(name, line_source) + parts = parse_req_from_line(pip_line, line_source) else: # It's a requirement - if "--index" in name: - name = name.split("--index")[0] - if " -i " in name: - name = name.split(" -i ")[0] + if "--index" in pip_line: + pip_line = pip_line.split("--index")[0] + if " -i " in pip_line: + pip_line = pip_line.split(" -i ")[0] # handle local version identifiers (like the ones torch uses in their public index) - if "+" in name: - name = name.split("+")[0] - parts = parse_req_from_line(name, line_source) + if "+" in pip_line: + pip_line = pip_line.split("+")[0] + parts = parse_req_from_line(pip_line, line_source) - return InstallRequirement( + install_req = InstallRequirement( parts.requirement, comes_from, link=parts.link, @@ -986,6 +1000,7 @@ def expansive_install_req_from_line( extras=parts.extras, user_supplied=user_supplied, ) + return install_req, name def _file_path_from_pipfile(path_obj, pipfile_entry): @@ -1068,7 +1083,7 @@ def install_req_from_pipfile(name, pipfile): version = "" req_str = f"{name}{extras_str}{version}" - install_req = expansive_install_req_from_line( + install_req, _ = expansive_install_req_from_line( req_str, comes_from=None, use_pep517=False, diff --git a/pipenv/utils/locking.py b/pipenv/utils/locking.py index c9552a57de..46e38d09af 100644 --- a/pipenv/utils/locking.py +++ b/pipenv/utils/locking.py @@ -454,7 +454,8 @@ def get_requirements( pip_line_specified = requirement_from_lockfile( package_name, package_info, include_hashes=True, include_markers=True ) - yield expansive_install_req_from_line(pip_line), pip_line_specified + install_req, _ = expansive_install_req_from_line(pip_line) + yield install_req, pip_line_specified def requirements_list(self, category: str) -> List[Dict]: if self.lockfile.get(category): diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index fc1a188b1e..1edbb25298 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -202,8 +202,11 @@ def create( if not dep: continue is_constraint = True - install_req = expansive_install_req_from_line(dep, expand_env=True) - package_name = determine_package_name(install_req) + install_req, package_name = expansive_install_req_from_line( + dep, expand_env=True + ) + if package_name is None: + package_name = determine_package_name(install_req) original_deps[package_name] = dep install_reqs[package_name] = install_req index, extra_index, trust_host, remainder = parse_indexes(dep)