Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Drop out-of-tree/in-tree build transition flags #11001

Merged
merged 1 commit into from
Apr 9, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions docs/html/topics/local-project-installs.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,17 @@ There are two advantages over using `setup.py develop` directly:
## Build artifacts

```{versionchanged} 21.3
The project being installed is no longer copied to a temporary directory before invoking the build system.
The project being installed is no longer copied to a temporary directory before invoking the build system, by default. A `--use-deprecated=out-of-tree-build` option is provided as a temporary fallback to aid user migrations.
```

This behaviour change has several consequences:
```{versionchanged} 22.1
The `--use-deprecated=out-of-tree-build` option has been removed.
```

When provided with a project that's in a local directory, pip will invoke the build system "in place". This behaviour has several consequences:

- Local project builds will now be significantly faster, for certain kinds of projects and on systems with slow I/O (eg: via network attached storage or overly aggressive antivirus software).
- Certain build backends (eg: `setuptools`) will litter the project directory with secondary build artifacts (eg: `.egg-info` directories).
- Certain build backends (eg: `setuptools`) may not be able to perform with parallel builds anymore, since they previously relied on the fact that pip invoked them in a separate directory for each build.

A `--use-deprecated=out-of-tree-build` option is available, until pip 22.1, as a mechanism to aid users with transitioning to the newer model of in-tree-builds.

[^1]: Specifically, the current machine's filesystem.
1 change: 1 addition & 0 deletions news/11001.removal.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Drop ``--use-deprecated=out-of-tree-build``, according to deprecation message.
3 changes: 1 addition & 2 deletions src/pip/_internal/cli/cmdoptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -957,7 +957,7 @@ def check_list_path_option(options: Values) -> None:
metavar="feature",
action="append",
default=[],
choices=["2020-resolver", "fast-deps", "in-tree-build"],
choices=["2020-resolver", "fast-deps"],
pradyunsg marked this conversation as resolved.
Show resolved Hide resolved
help="Enable new functionality, that may be backward incompatible.",
)

Expand All @@ -970,7 +970,6 @@ def check_list_path_option(options: Values) -> None:
default=[],
choices=[
"legacy-resolver",
"out-of-tree-build",
"backtrack-on-build-failures",
"html5lib",
],
Expand Down
15 changes: 0 additions & 15 deletions src/pip/_internal/cli/req_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,20 +288,6 @@ def make_requirement_preparer(
"fast-deps has no effect when used with the legacy resolver."
)

in_tree_build = "out-of-tree-build" not in options.deprecated_features_enabled
if "in-tree-build" in options.features_enabled:
deprecated(
reason="In-tree builds are now the default.",
replacement="to remove the --use-feature=in-tree-build flag",
gone_in="22.1",
)
if "out-of-tree-build" in options.deprecated_features_enabled:
deprecated(
reason="Out-of-tree builds are deprecated.",
replacement=None,
gone_in="22.1",
)

return RequirementPreparer(
build_dir=temp_build_dir_path,
src_dir=options.src_dir,
Expand All @@ -315,7 +301,6 @@ def make_requirement_preparer(
use_user_site=use_user_site,
lazy_wheel=lazy_wheel,
verbosity=verbosity,
in_tree_build=in_tree_build,
)

@classmethod
Expand Down
74 changes: 4 additions & 70 deletions src/pip/_internal/operations/prepare.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,9 @@
from pip._internal.network.session import PipSession
from pip._internal.operations.build.build_tracker import BuildTracker
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.filesystem import copy2_fixed
from pip._internal.utils.hashes import Hashes, MissingHashes
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree
from pip._internal.utils.misc import display_path, hide_url, is_installable_dir
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.unpacking import unpack_file
from pip._internal.vcs import vcs
Expand Down Expand Up @@ -98,55 +97,6 @@ def get_http_url(
return File(from_path, content_type)


def _copy2_ignoring_special_files(src: str, dest: str) -> None:
"""Copying special files is not supported, but as a convenience to users
we skip errors copying them. This supports tools that may create e.g.
socket files in the project source directory.
"""
try:
copy2_fixed(src, dest)
except shutil.SpecialFileError as e:
# SpecialFileError may be raised due to either the source or
# destination. If the destination was the cause then we would actually
# care, but since the destination directory is deleted prior to
# copy we ignore all of them assuming it is caused by the source.
logger.warning(
"Ignoring special file error '%s' encountered copying %s to %s.",
str(e),
src,
dest,
)


def _copy_source_tree(source: str, target: str) -> None:
target_abspath = os.path.abspath(target)
target_basename = os.path.basename(target_abspath)
target_dirname = os.path.dirname(target_abspath)

def ignore(d: str, names: List[str]) -> List[str]:
skipped: List[str] = []
if d == source:
# Pulling in those directories can potentially be very slow,
# exclude the following directories if they appear in the top
# level dir (and only it).
# See discussion at https://github.com/pypa/pip/pull/6770
skipped += [".tox", ".nox"]
if os.path.abspath(d) == target_dirname:
# Prevent an infinite recursion if the target is in source.
# This can happen when TMPDIR is set to ${PWD}/...
# and we copy PWD to TMPDIR.
skipped += [target_basename]
return skipped

shutil.copytree(
source,
target,
ignore=ignore,
symlinks=True,
copy_function=_copy2_ignoring_special_files,
pradyunsg marked this conversation as resolved.
Show resolved Hide resolved
)


def get_file_url(
link: Link, download_dir: Optional[str] = None, hashes: Optional[Hashes] = None
) -> File:
Expand Down Expand Up @@ -191,19 +141,7 @@ def unpack_url(
unpack_vcs_link(link, location, verbosity=verbosity)
return None

# Once out-of-tree-builds are no longer supported, could potentially
# replace the below condition with `assert not link.is_existing_dir`
# - unpack_url does not need to be called for in-tree-builds.
#
# As further cleanup, _copy_source_tree and accompanying tests can
# be removed.
#
# TODO when use-deprecated=out-of-tree-build is removed
if link.is_existing_dir():
if os.path.isdir(location):
rmtree(location)
_copy_source_tree(link.file_path, location)
return None
assert not link.is_existing_dir()

# file urls
if link.is_file:
Expand Down Expand Up @@ -269,7 +207,6 @@ def __init__(
use_user_site: bool,
lazy_wheel: bool,
verbosity: int,
in_tree_build: bool,
) -> None:
super().__init__()

Expand Down Expand Up @@ -300,9 +237,6 @@ def __init__(
# How verbose should underlying tooling be?
self.verbosity = verbosity

# Should in-tree builds be used for local paths?
self.in_tree_build = in_tree_build

# Memoized downloaded files, as mapping of url: path.
self._downloaded: Dict[str, str] = {}

Expand Down Expand Up @@ -336,7 +270,7 @@ def _ensure_link_req_src_dir(
# directory.
return
assert req.source_dir is None
if req.link.is_existing_dir() and self.in_tree_build:
if req.link.is_existing_dir():
# build local directories in-tree
req.source_dir = req.link.file_path
return
Expand Down Expand Up @@ -525,7 +459,7 @@ def _prepare_linked_requirement(
self._ensure_link_req_src_dir(req, parallel_builds)
hashes = self._get_linked_req_hashes(req)

if link.is_existing_dir() and self.in_tree_build:
if link.is_existing_dir():
local_file = None
elif link.url not in self._downloaded:
try:
Expand Down
29 changes: 0 additions & 29 deletions src/pip/_internal/utils/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
import os
import os.path
import random
import shutil
import stat
import sys
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
Expand Down Expand Up @@ -42,33 +40,6 @@ def check_path_owner(path: str) -> bool:
return False # assume we don't own the path


def copy2_fixed(src: str, dest: str) -> None:
"""Wrap shutil.copy2() but map errors copying socket files to
SpecialFileError as expected.

See also https://bugs.python.org/issue37700.
"""
try:
shutil.copy2(src, dest)
except OSError:
for f in [src, dest]:
try:
is_socket_file = is_socket(f)
except OSError:
# An error has already occurred. Another error here is not
# a problem and we can ignore it.
pass
else:
if is_socket_file:
raise shutil.SpecialFileError(f"`{f}` is a socket")

raise


def is_socket(path: str) -> bool:
return stat.S_ISSOCK(os.lstat(path).st_mode)


@contextmanager
def adjacent_tmp_file(path: str, **kwargs: Any) -> Iterator[BinaryIO]:
"""Return a file-like object pointing to a tmp file next to path.
Expand Down
54 changes: 0 additions & 54 deletions tests/functional/test_install.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import glob
import os
import re
import shutil
import ssl
import sys
import textwrap
Expand Down Expand Up @@ -30,7 +29,6 @@
pyversion,
requirements_file,
)
from tests.lib.filesystem import make_socket_file
from tests.lib.local_repos import local_checkout
from tests.lib.path import Path
from tests.lib.server import (
Expand Down Expand Up @@ -648,26 +646,6 @@ def test_hashed_install_failure_later_flag(
)


@pytest.mark.usefixtures("with_wheel")
def test_install_from_local_directory_with_symlinks_to_directories(
script: PipTestEnvironment, data: TestData
) -> None:
"""
Test installing from a local directory containing symlinks to directories.
"""
to_install = data.packages.joinpath("symlinks")
result = script.pip(
"install",
"--use-deprecated=out-of-tree-build",
to_install,
allow_stderr_warning=True, # TODO: set to False when removing out-of-tree-build
)
pkg_folder = script.site_packages / "symlinks"
dist_info_folder = script.site_packages / "symlinks-0.1.dev0.dist-info"
result.did_create(pkg_folder)
result.did_create(dist_info_folder)


@pytest.mark.usefixtures("with_wheel")
def test_install_from_local_directory_with_in_tree_build(
script: PipTestEnvironment, data: TestData
Expand All @@ -688,38 +666,6 @@ def test_install_from_local_directory_with_in_tree_build(
assert in_tree_build_dir.exists()


@pytest.mark.skipif("sys.platform == 'win32'")
@pytest.mark.usefixtures("with_wheel")
def test_install_from_local_directory_with_socket_file(
script: PipTestEnvironment, data: TestData, tmpdir: Path
) -> None:
"""
Test installing from a local directory containing a socket file.
"""
# TODO: remove this test when removing out-of-tree-build support,
# it is only meant to test the copy of socket files
dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info"
package_folder = script.site_packages / "fspkg"
to_copy = data.packages.joinpath("FSPkg")
to_install = tmpdir.joinpath("src")

shutil.copytree(to_copy, to_install)
# Socket file, should be ignored.
socket_file_path = os.path.join(to_install, "example")
make_socket_file(socket_file_path)

result = script.pip(
"install",
"--use-deprecated=out-of-tree-build",
"--verbose",
to_install,
allow_stderr_warning=True, # because of the out-of-tree deprecation warning
)
result.did_create(package_folder)
result.did_create(dist_info_folder)
assert str(socket_file_path) in result.stderr


def test_install_from_local_directory_with_no_setup_py(
script: PipTestEnvironment, data: TestData
) -> None:
Expand Down
28 changes: 0 additions & 28 deletions tests/lib/filesystem.py
Original file line number Diff line number Diff line change
@@ -1,38 +1,10 @@
"""Helpers for filesystem-dependent tests.
"""
import os
import socket
import subprocess
import sys
from functools import partial
from itertools import chain
from typing import Iterator, List, Set

from .path import Path


def make_socket_file(path: str) -> None:
# Socket paths are limited to 108 characters (sometimes less) so we
# chdir before creating it and use a relative path name.
cwd = os.getcwd()
os.chdir(os.path.dirname(path))
try:
sock = socket.socket(socket.AF_UNIX)
sock.bind(os.path.basename(path))
finally:
os.chdir(cwd)


def make_unreadable_file(path: str) -> None:
Path(path).touch()
os.chmod(path, 0o000)
if sys.platform == "win32":
username = os.getlogin()
# Remove "Read Data/List Directory" permission for current user, but
# leave everything else.
args = ["icacls", path, "/deny", username + ":(RD)"]
subprocess.check_call(args)


def get_filelist(base: str) -> Set[str]:
def join(dirpath: str, dirnames: List[str], filenames: List[str]) -> Iterator[str]:
Expand Down
Loading