Merge branch 'pypa:main' into joe/warm_cache_in_threadpool

.github/workflows/ci.yml

@@ -167,24 +167,13 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
-      # We use a RAMDisk on Windows, since filesystem IO is a big slowdown
-      # for our tests.
-      - name: Create a RAMDisk
-        run: ./tools/ci/New-RAMDisk.ps1 -Drive R -Size 1GB
-
-      - name: Setup RAMDisk permissions
-        run: |
-          mkdir R:\Temp
-          $acl = Get-Acl "R:\Temp"
-          $rule = New-Object System.Security.AccessControl.FileSystemAccessRule(
-              "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow"
-          )
-          $acl.AddAccessRule($rule)
-          Set-Acl "R:\Temp" $acl
-
+      # We use C:\Temp (which is already available on the worker)
+      # as a temporary directory for all of the tests because the
+      # default value (under the user dir) is more deeply nested
+      # and causes tests to fail with "path too long" errors.
       - run: pip install nox
         env:
-          TEMP: "R:\\Temp"
+          TEMP: "C:\\Temp"
 
       # Main check
       - name: Run unit tests
@@ -194,7 +183,7 @@ jobs:
           -m unit
           --verbose --numprocesses auto --showlocals
         env:
-          TEMP: "R:\\Temp"
+          TEMP: "C:\\Temp"
 
       - name: Run integration tests (group 1)
         if: matrix.group == 1
@@ -203,7 +192,7 @@ jobs:
           -m integration -k "not test_install"
           --verbose --numprocesses auto --showlocals
         env:
-          TEMP: "R:\\Temp"
+          TEMP: "C:\\Temp"
 
       - name: Run integration tests (group 2)
         if: matrix.group == 2
@@ -212,7 +201,7 @@ jobs:
           -m integration -k "test_install"
           --verbose --numprocesses auto --showlocals
         env:
-          TEMP: "R:\\Temp"
+          TEMP: "C:\\Temp"
 
   tests-zipapp:
     name: tests / zipapp

.pre-commit-config.yaml

@@ -21,22 +21,11 @@ repos:
   hooks:
   - id: black
 
-- repo: https://github.com/PyCQA/flake8
-  rev: 6.0.0
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.0.270
   hooks:
-  - id: flake8
-    additional_dependencies: [
-        'flake8-bugbear',
-        'flake8-logging-format',
-        'flake8-implicit-str-concat',
-    ]
-    exclude: tests/data
-
-- repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
-  hooks:
-  - id: isort
-    files: \.py$
+    - id: ruff
 
 - repo: https://github.com/pre-commit/mirrors-mypy
   rev: v0.961

NEWS.rst

@@ -53,7 +53,7 @@ Deprecations and Removals
   ``--config-settings``. (`#11859 <https://github.com/pypa/pip/issues/11859>`_)
 - Using ``--config-settings`` with projects that don't have a ``pyproject.toml`` now prints
   a deprecation warning. In the future the presence of config settings will automatically
-  enable the default build backend for legacy projects and pass the setttings to it. (`#11915 <https://github.com/pypa/pip/issues/11915>`_)
+  enable the default build backend for legacy projects and pass the settings to it. (`#11915 <https://github.com/pypa/pip/issues/11915>`_)
 - Remove ``setup.py install`` fallback when building a wheel failed for projects without
   ``pyproject.toml``. (`#8368 <https://github.com/pypa/pip/issues/8368>`_)
 - When the ``wheel`` package is not installed, pip now uses the default build backend

news/11847.bugfix.rst

@@ -0,0 +1 @@
+Prevent downloading files twice when PEP 658 metadata is present

news/11920.bugfix.rst

@@ -0,0 +1 @@
+Add permission check before configuration

news/11996.process.rst

@@ -0,0 +1 @@
+Deprecate support for eggs for Python 3.11 or later, when the new ``importlib.metadata`` backend is used to load distribution metadata. This only affects the egg *distribution format* (with the ``.egg`` extension); distributions using the ``.egg-info`` *metadata format* (but are not actually eggs) are not affected. For more information about eggs, see `relevant section in the setuptools documentation <https://setuptools.pypa.io/en/stable/deprecated/python_eggs.html>`__.

news/12042.bugfix.rst

@@ -0,0 +1 @@
+Correctly parse ``dist-info-metadata`` values from JSON-format index data.

news/12063.removal.rst

@@ -0,0 +1,2 @@
+Deprecate legacy version and version specifiers that don't conform to `PEP 440
+<https://peps.python.org/pep-0440/>`_

news/12067.bugfix.rst

@@ -0,0 +1 @@
+Fail with an error if the ``--python`` option is specified after the subcommand name.

news/12079.bugfix.rst

@@ -0,0 +1 @@
+Fix slowness when using ``importlib.metadata`` (the default way for pip to read metadata in Python 3.11+) and there is a large overlap between already installed and to-be-installed packages.

news/12119.bugfix.rst

@@ -0,0 +1,3 @@
+Pass the ``-r`` flag to mercurial to be explicit that a revision is passed and protect
+against ``hg`` options injection as part of VCS URLs. Users that do not have control on
+VCS URLs passed to pip are advised to upgrade.

news/4256.removal.rst

@@ -0,0 +1,4 @@
+``freeze`` no longer excludes the ``setuptools``, ``distribute``, and ``wheel``
+from the output when running on Python 3.12 or later, where they are not
+included in a virtual environment by default. Use ``--exclude`` if you wish to
+exclude any of these packages.

news/certifi.vendor.rst

@@ -0,0 +1 @@
+Upgrade certifi to 2023.5.7

news/platformdirs.vendor.rst

@@ -0,0 +1 @@
+Upgrade platformdirs to 3.8.1

news/pygments.vendor.rst

@@ -0,0 +1 @@
+Upgrade pygments to 2.15.1

news/pyparsing.vendor.rst

@@ -0,0 +1 @@
+Upgrade pyparsing to 3.1.0

news/rich.vendor.rst

@@ -0,0 +1 @@
+Upgrade rich to 13.4.2

news/setuptools.vendor.rst

@@ -0,0 +1 @@
+Upgrade setuptools to 68.0.0

news/typing_extensions.vendor.rst

@@ -0,0 +1 @@
+Upgrade typing_extensions to 4.7.1

news/urllib3.vendor.rst

@@ -0,0 +1 @@
+Upgrade urllib3 to 1.26.16

noxfile.py

@@ -219,7 +219,7 @@ def pinned_requirements(path: Path) -> Iterator[Tuple[str, str]]:
         new_version = old_version
         for inner_name, inner_version in pinned_requirements(vendor_txt):
             if inner_name == name:
-                # this is a dedicated assignment, to make flake8 happy
+                # this is a dedicated assignment, to make lint happy
                 new_version = inner_version
                 break
         else:

pyproject.toml

@@ -71,3 +71,38 @@ setuptools = "pkg_resources"
 CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt"
 distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt"
 webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE"
+
+[tool.ruff]
+extend-exclude = [
+    "./build",
+    ".scratch",
+    "_vendor",
+    "data",
+]
+ignore = [
+    "B019",
+    "B020",
+    "B904", # Ruff enables opinionated warnings by default
+    "B905", # Ruff enables opinionated warnings by default
+    "G202",
+]
+line-length = 88
+select = [
+    "B",
+    "E",
+    "F",
+    "W",
+    "G",
+    "ISC",
+    "I",
+]
+
+[tool.ruff.per-file-ignores]
+"noxfile.py" = ["G"]
+"tests/*" = ["B011"]
+
+[tool.ruff.isort]
+# We need to explicitly make pip "first party" as it's imported by code in
+# the docs and tests directories.
+known-first-party = ["pip"]
+known-third-party = ["pip._vendor"]

src/pip/_internal/cli/base_command.py

@@ -131,6 +131,17 @@ def _main(self, args: List[str]) -> int:
                 ", ".join(sorted(always_enabled_features)),
             )
 
+        # Make sure that the --python argument isn't specified after the
+        # subcommand. We can tell, because if --python was specified,
+        # we should only reach this point if we're running in the created
+        # subprocess, which has the _PIP_RUNNING_IN_SUBPROCESS environment
+        # variable set.
+        if options.python and "_PIP_RUNNING_IN_SUBPROCESS" not in os.environ:
+            logger.critical(
+                "The --python option must be placed before the pip subcommand name"
+            )
+            sys.exit(ERROR)
+
         # TODO: Try to get these passing down from the command?
         #       without resorting to os.environ to hold these.
         #       This also affects isolated builds and it should.

src/pip/_internal/commands/check.py

@@ -7,6 +7,7 @@
 from pip._internal.operations.check import (
     check_package_set,
     create_package_set_from_installed,
+    warn_legacy_versions_and_specifiers,
 )
 from pip._internal.utils.misc import write_output
 
@@ -21,6 +22,7 @@ class CheckCommand(Command):
 
     def run(self, options: Values, args: List[str]) -> int:
         package_set, parsing_probs = create_package_set_from_installed()
+        warn_legacy_versions_and_specifiers(package_set)
         missing, conflicting = check_package_set(package_set)
 
         for project_name in missing:

src/pip/_internal/commands/download.py

@@ -130,6 +130,7 @@ def run(self, options: Values, args: List[str]) -> int:
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        requirement_set.warn_legacy_versions_and_specifiers()
 
         downloaded: List[str] = []
         for req in requirement_set.requirements.values():

src/pip/_internal/commands/freeze.py

@@ -1,14 +1,25 @@
 import sys
 from optparse import Values
-from typing import List
+from typing import AbstractSet, List
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import SUCCESS
 from pip._internal.operations.freeze import freeze
 from pip._internal.utils.compat import stdlib_pkgs
 
-DEV_PKGS = {"pip", "setuptools", "distribute", "wheel"}
+
+def _should_suppress_build_backends() -> bool:
+    return sys.version_info < (3, 12)
+
+
+def _dev_pkgs() -> AbstractSet[str]:
+    pkgs = {"pip"}
+
+    if _should_suppress_build_backends():
+        pkgs |= {"setuptools", "distribute", "wheel"}
+
+    return pkgs
 
 
 class FreezeCommand(Command):
@@ -61,7 +72,7 @@ def add_options(self) -> None:
             action="store_true",
             help=(
                 "Do not skip these packages in the output:"
-                " {}".format(", ".join(DEV_PKGS))
+                " {}".format(", ".join(_dev_pkgs()))
             ),
         )
         self.cmd_opts.add_option(
@@ -77,7 +88,7 @@ def add_options(self) -> None:
     def run(self, options: Values, args: List[str]) -> int:
         skip = set(stdlib_pkgs)
         if not options.freeze_all:
-            skip.update(DEV_PKGS)
+            skip.update(_dev_pkgs())
 
         if options.excludes:
             skip.update(options.excludes)

src/pip/_internal/commands/install.py

@@ -387,6 +387,9 @@ def run(self, options: Values, args: List[str]) -> int:
                         json.dump(report.to_dict(), f, indent=2, ensure_ascii=False)
 
             if options.dry_run:
+                # In non dry-run mode, the legacy versions and specifiers check
+                # will be done as part of conflict detection.
+                requirement_set.warn_legacy_versions_and_specifiers()
                 would_install_items = sorted(
                     (r.metadata["name"], r.metadata["version"])
                     for r in requirement_set.requirements_to_install

src/pip/_internal/commands/wheel.py

@@ -145,6 +145,7 @@ def run(self, options: Values, args: List[str]) -> int:
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        requirement_set.warn_legacy_versions_and_specifiers()
 
         reqs_to_build: List[InstallRequirement] = []
         for req in requirement_set.requirements.values():

src/pip/_internal/configuration.py

@@ -210,8 +210,15 @@ def save(self) -> None:
             # Ensure directory exists.
             ensure_dir(os.path.dirname(fname))
 
-            with open(fname, "w") as f:
-                parser.write(f)
+            # Ensure directory's permission(need to be writeable)
+            try:
+                with open(fname, "w") as f:
+                    parser.write(f)
+            except OSError as error:
+                raise ConfigurationError(
+                    f"An error occurred while writing to the configuration file "
+                    f"{fname}: {error}"
+                )
 
     #
     # Private routines

src/pip/_internal/metadata/importlib/_envs.py

@@ -151,7 +151,7 @@ def _emit_egg_deprecation(location: Optional[str]) -> None:
     deprecated(
         reason=f"Loading egg at {location} is deprecated.",
         replacement="to use pip for package installation.",
-        gone_in=None,
+        gone_in="23.3",
     )
 
 
@@ -174,7 +174,7 @@ def _iter_distributions(self) -> Iterator[BaseDistribution]:
         for location in self._paths:
             yield from finder.find(location)
             for dist in finder.find_eggs(location):
-                # _emit_egg_deprecation(dist.location)  # TODO: Enable this.
+                _emit_egg_deprecation(dist.location)
                 yield dist
             # This must go last because that's how pkg_resources tie-breaks.
             yield from finder.find_linked(location)