From 2fb4874712368644967c5628db31a49aea9f7aa0 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Sat, 27 Feb 2021 13:34:33 -0800 Subject: [PATCH 01/25] Support custom python-executable --- piptools/scripts/sync.py | 22 ++++++++++++++++++++-- piptools/sync.py | 7 +++++-- piptools/utils.py | 16 ++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 56fe3050c..1a124515d 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -15,7 +15,7 @@ from ..exceptions import PipToolsError from ..logging import log from ..repositories import PyPIRepository -from ..utils import flat_map +from ..utils import flat_map, get_sys_path_for_python_executable DEFAULT_REQUIREMENTS_FILE = "requirements.txt" @@ -55,6 +55,10 @@ is_flag=True, help="Ignore package index (only looking at --find-links URLs instead)", ) +@click.option( + "--python-executable", + help="Custom python executable if targeting an environment other than current", +) @click.option("-v", "--verbose", count=True, help="Show more output") @click.option("-q", "--quiet", count=True, help="Give less output") @click.option( @@ -77,6 +81,7 @@ def cli( extra_index_url: Tuple[str, ...], trusted_host: Tuple[str, ...], no_index: bool, + python_executable: Optional[str], verbose: int, quiet: int, user_only: bool, @@ -108,6 +113,12 @@ def cli( log.error("ERROR: " + msg) sys.exit(2) + if python_executable is not None: + if not os.path.exists(python_executable): + msg = "Python executable {} not found" + log.error(msg.format(python_executable)) + sys.exit(2) + install_command = cast(InstallCommand, create_command("install")) options, _ = install_command.parse_args([]) session = install_command._build_session(options) @@ -125,7 +136,13 @@ def cli( log.error(str(e)) sys.exit(2) - installed_dists = get_installed_distributions(skip=[], user_only=user_only) + paths = None + if python_executable: + paths = get_sys_path_for_python_executable(python_executable) + + installed_dists = get_installed_distributions( + skip=[], user_only=user_only, paths=paths, local_only=python_executable is None + ) to_install, to_uninstall = sync.diff(merged_requirements, installed_dists) install_flags = ( @@ -149,6 +166,7 @@ def cli( dry_run=dry_run, install_flags=install_flags, ask=ask, + python_executable=python_executable, ) ) diff --git a/piptools/sync.py b/piptools/sync.py index e9d25bbe0..ed599e2e9 100644 --- a/piptools/sync.py +++ b/piptools/sync.py @@ -178,12 +178,15 @@ def sync( dry_run: bool = False, install_flags: Optional[List[str]] = None, ask: bool = False, + python_executable: Optional[str] = None, ) -> int: """ Install and uninstalls the given sets of modules. """ exit_code = 0 + python_executable = python_executable or sys.executable + if not to_uninstall and not to_install: log.info("Everything up-to-date", err=False) return exit_code @@ -216,7 +219,7 @@ def sync( if to_uninstall: run( # nosec [ - sys.executable, + python_executable, "-m", "pip", "uninstall", @@ -244,7 +247,7 @@ def sync( try: run( # nosec [ - sys.executable, + python_executable, "-m", "pip", "install", diff --git a/piptools/utils.py b/piptools/utils.py index cde26a5e1..05511fa3c 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -1,11 +1,14 @@ import collections import itertools +import json import shlex +import subprocess # nosec from typing import ( Callable, Dict, Iterable, Iterator, + List, Optional, Set, Tuple, @@ -324,3 +327,16 @@ def get_compile_command(click_ctx: click.Context) -> str: left_args.append(f"{option_long_name}={shlex.quote(str(val))}") return " ".join(["pip-compile", *sorted(left_args), *sorted(right_args)]) + + +def get_sys_path_for_python_executable(python_executable: str) -> List[str]: + """ + Returns sys.path list for given python executable. + """ + result = subprocess.check_output( # nosec + [python_executable, "-c", "import sys;import json;print(json.dumps(sys.path))"] + ) + paths = json.loads(result) + assert isinstance(paths, list) + assert all(isinstance(i, str) for i in paths) + return paths From 5a68984d7342a38cb30951aa0f01798f2ef7a60d Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Mon, 8 Mar 2021 17:44:46 -0800 Subject: [PATCH 02/25] add tests --- piptools/utils.py | 2 +- tests/test_cli_sync.py | 52 ++++++++++++++++++++++++++++++++++++++++++ tests/test_utils.py | 7 ++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index 05511fa3c..c24c4b473 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -331,7 +331,7 @@ def get_compile_command(click_ctx: click.Context) -> str: def get_sys_path_for_python_executable(python_executable: str) -> List[str]: """ - Returns sys.path list for given python executable. + Returns sys.path list for the given python executable. """ result = subprocess.check_output( # nosec [python_executable, "-c", "import sys;import json;print(json.dumps(sys.path))"] diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index fe4fc94e1..e52849935 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -242,3 +242,55 @@ def test_sync_dry_run_returns_non_zero_exit_code(runner): out = runner.invoke(cli, ["--dry-run"]) assert out.exit_code == 1 + + +@mock.patch("piptools.scripts.sync.get_sys_path_for_python_executable") +@mock.patch("piptools.scripts.sync.get_installed_distributions") +@mock.patch("piptools.sync.run") +def test_python_executable_option(run, get_installed_distributions, get_sys_path_for_python_executable, runner, fake_dist): + """ + Make sure sync command can run with `--python-executable` option + """ + with open("requirements.txt", "w") as req_in: + req_in.write("small-fake-a==1.10.0") + + custom_executable = 'custom_executable' + with open(custom_executable, "w") as exec_file: + exec_file.write("") + + sys_paths = ['', './'] + get_sys_path_for_python_executable.return_value = sys_paths + get_installed_distributions.return_value = [fake_dist("django==1.8")] + + runner.invoke(cli, ["--python-executable", custom_executable]) + + get_installed_distributions.assert_called_once_with( + skip=[], + user_only=False, + paths=sys_paths, + local_only=False + ) + + assert run.call_count == 2 + fist_call, second_call = run.mock_calls + assert fist_call == mock.call([custom_executable, '-m', 'pip', 'uninstall', '-y', 'django'], check=True) + assert second_call.args[0][:-1] ==[custom_executable, '-m', 'pip', 'install', '-r'] + + +def test_invalid_python_executable(runner): + out = runner.invoke(cli, ["--python-executable", "/tmp/invalid_executable"]) + assert out.exit_code == 2, out + + +@mock.patch("piptools.sync.run") +def test_default_python_executable_option(run, runner): + """ + Make sure sys.executable is used when --python-executable is not provided + """ + with open("requirements.txt", "w") as req_in: + req_in.write("small-fake-a==1.10.0") + + runner.invoke(cli) + + assert run.call_count == 2 + assert run.mock_calls[1].args[0][:-1] == [sys.executable, '-m', 'pip', 'install', '-r'] \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 508307170..cd0facb89 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,6 +2,7 @@ import operator import os import shlex +import sys import pytest @@ -14,6 +15,7 @@ format_specifier, get_compile_command, get_hashes_from_ireq, + get_sys_path_for_python_executable, is_pinned_requirement, is_url_requirement, lookup_table, @@ -406,3 +408,8 @@ def test_lookup_table_requires_key(values): ) def test_lookup_table_with_empty_values(key): assert lookup_table((), key) == {} + + +def test_get_sys_path_for_python_executable(): + result = get_sys_path_for_python_executable(sys.executable) + assert result == [''] + sys.path From 1c321cde7675a7d44596d9e02003a4917c47c9d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Mar 2021 01:50:37 +0000 Subject: [PATCH 03/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_cli_sync.py | 31 +++++++++++++++++++++---------- tests/test_utils.py | 2 +- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index e52849935..f7ff6f62f 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -247,34 +247,39 @@ def test_sync_dry_run_returns_non_zero_exit_code(runner): @mock.patch("piptools.scripts.sync.get_sys_path_for_python_executable") @mock.patch("piptools.scripts.sync.get_installed_distributions") @mock.patch("piptools.sync.run") -def test_python_executable_option(run, get_installed_distributions, get_sys_path_for_python_executable, runner, fake_dist): +def test_python_executable_option( + run, + get_installed_distributions, + get_sys_path_for_python_executable, + runner, + fake_dist, +): """ Make sure sync command can run with `--python-executable` option """ with open("requirements.txt", "w") as req_in: req_in.write("small-fake-a==1.10.0") - custom_executable = 'custom_executable' + custom_executable = "custom_executable" with open(custom_executable, "w") as exec_file: exec_file.write("") - sys_paths = ['', './'] + sys_paths = ["", "./"] get_sys_path_for_python_executable.return_value = sys_paths get_installed_distributions.return_value = [fake_dist("django==1.8")] runner.invoke(cli, ["--python-executable", custom_executable]) get_installed_distributions.assert_called_once_with( - skip=[], - user_only=False, - paths=sys_paths, - local_only=False + skip=[], user_only=False, paths=sys_paths, local_only=False ) assert run.call_count == 2 fist_call, second_call = run.mock_calls - assert fist_call == mock.call([custom_executable, '-m', 'pip', 'uninstall', '-y', 'django'], check=True) - assert second_call.args[0][:-1] ==[custom_executable, '-m', 'pip', 'install', '-r'] + assert fist_call == mock.call( + [custom_executable, "-m", "pip", "uninstall", "-y", "django"], check=True + ) + assert second_call.args[0][:-1] == [custom_executable, "-m", "pip", "install", "-r"] def test_invalid_python_executable(runner): @@ -293,4 +298,10 @@ def test_default_python_executable_option(run, runner): runner.invoke(cli) assert run.call_count == 2 - assert run.mock_calls[1].args[0][:-1] == [sys.executable, '-m', 'pip', 'install', '-r'] \ No newline at end of file + assert run.mock_calls[1].args[0][:-1] == [ + sys.executable, + "-m", + "pip", + "install", + "-r", + ] diff --git a/tests/test_utils.py b/tests/test_utils.py index 9845976fa..678aedeb1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -373,4 +373,4 @@ def test_lookup_table_with_empty_values(): def test_get_sys_path_for_python_executable(): result = get_sys_path_for_python_executable(sys.executable) - assert result == [''] + sys.path \ No newline at end of file + assert result == [""] + sys.path From a3fd7a5f963bf6f033d5ed5a521b7dafcf808500 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 16 Mar 2021 09:46:53 -0700 Subject: [PATCH 04/25] fix test --- piptools/utils.py | 3 ++- tests/test_utils.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 24de63677..3fc3379a2 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -1,6 +1,7 @@ import collections import itertools import json +import os import shlex import subprocess # nosec from typing import ( @@ -321,4 +322,4 @@ def get_sys_path_for_python_executable(python_executable: str) -> List[str]: paths = json.loads(result) assert isinstance(paths, list) assert all(isinstance(i, str) for i in paths) - return paths + return [os.path.abspath(path) for path in paths] diff --git a/tests/test_utils.py b/tests/test_utils.py index 678aedeb1..250f3873f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -373,4 +373,7 @@ def test_lookup_table_with_empty_values(): def test_get_sys_path_for_python_executable(): result = get_sys_path_for_python_executable(sys.executable) - assert result == [""] + sys.path + # not testing for equality, because pytest adds extra paths into current sys.path + for path in result: + assert path in sys.path + From fc4b11ebda0579e5084628bbe4aa0abe1da66c86 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Mar 2021 16:47:18 +0000 Subject: [PATCH 05/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 250f3873f..50434a95c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -376,4 +376,3 @@ def test_get_sys_path_for_python_executable(): # not testing for equality, because pytest adds extra paths into current sys.path for path in result: assert path in sys.path - From 7cde62d81967d96c7dce53ee7ba539bd128e1cbb Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 16 Mar 2021 10:06:46 -0700 Subject: [PATCH 06/25] fix tests for older python versions --- tests/test_cli_sync.py | 31 ++++++++++++++++++++----------- tests/test_utils.py | 1 - 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index f7ff6f62f..8024ed2f9 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -275,11 +275,15 @@ def test_python_executable_option( ) assert run.call_count == 2 - fist_call, second_call = run.mock_calls - assert fist_call == mock.call( - [custom_executable, "-m", "pip", "uninstall", "-y", "django"], check=True - ) - assert second_call.args[0][:-1] == [custom_executable, "-m", "pip", "install", "-r"] + + call_args = [call[0][0] for call in run.call_args_list] + called_uninstall_options = [args for args in call_args if args[3] == "uninstall"] + called_install_options = [args[:-1] for args in call_args if args[3] == "install"] + + assert called_uninstall_options == [ + [custom_executable, "-m", "pip", "uninstall", "-y", "django"] + ] + assert called_install_options == [[custom_executable, "-m", "pip", "install", "-r"]] def test_invalid_python_executable(runner): @@ -298,10 +302,15 @@ def test_default_python_executable_option(run, runner): runner.invoke(cli) assert run.call_count == 2 - assert run.mock_calls[1].args[0][:-1] == [ - sys.executable, - "-m", - "pip", - "install", - "-r", + + call_args = [call[0][0] for call in run.call_args_list] + called_install_options = [args[:-1] for args in call_args if args[3] == "install"] + assert called_install_options == [ + [ + sys.executable, + "-m", + "pip", + "install", + "-r", + ] ] diff --git a/tests/test_utils.py b/tests/test_utils.py index 250f3873f..50434a95c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -376,4 +376,3 @@ def test_get_sys_path_for_python_executable(): # not testing for equality, because pytest adds extra paths into current sys.path for path in result: assert path in sys.path - From 786671aa64addc69d3d4369537e2cb5c1d5df445 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 16 Mar 2021 10:24:49 -0700 Subject: [PATCH 07/25] code coverage --- tests/test_cli_sync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 8024ed2f9..b78adb0c0 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -287,6 +287,9 @@ def test_python_executable_option( def test_invalid_python_executable(runner): + with open("requirements.txt", "w") as req_in: + req_in.write("small-fake-a==1.10.0") + out = runner.invoke(cli, ["--python-executable", "/tmp/invalid_executable"]) assert out.exit_code == 2, out From c25dc6ff1db89564fcdfa3d0b577fb43094b0555 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 16 Mar 2021 10:58:29 -0700 Subject: [PATCH 08/25] comsetic updates --- piptools/scripts/sync.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 1a124515d..37622e54f 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -57,7 +57,7 @@ ) @click.option( "--python-executable", - help="Custom python executable if targeting an environment other than current", + help="Custom python executable path if targeting an environment other than current", ) @click.option("-v", "--verbose", count=True, help="Show more output") @click.option("-q", "--quiet", count=True, help="Give less output") @@ -113,11 +113,10 @@ def cli( log.error("ERROR: " + msg) sys.exit(2) - if python_executable is not None: - if not os.path.exists(python_executable): - msg = "Python executable {} not found" - log.error(msg.format(python_executable)) - sys.exit(2) + if python_executable is not None and not os.path.exists(python_executable): + msg = "Python executable {} not found" + log.error(msg.format(python_executable)) + sys.exit(2) install_command = cast(InstallCommand, create_command("install")) options, _ = install_command.parse_args([]) From 9733491fc2bc13dee7cc037a713a4434d8ebc896 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 16 Mar 2021 22:37:31 -0700 Subject: [PATCH 09/25] use type=click.Path --- piptools/scripts/sync.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 37622e54f..0996f1b09 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -57,6 +57,7 @@ ) @click.option( "--python-executable", + type=click.Path(exists=True), help="Custom python executable path if targeting an environment other than current", ) @click.option("-v", "--verbose", count=True, help="Show more output") @@ -113,11 +114,6 @@ def cli( log.error("ERROR: " + msg) sys.exit(2) - if python_executable is not None and not os.path.exists(python_executable): - msg = "Python executable {} not found" - log.error(msg.format(python_executable)) - sys.exit(2) - install_command = cast(InstallCommand, create_command("install")) options, _ = install_command.parse_args([]) session = install_command._build_session(options) From b01feb5bef90c589aa634748af48b4a0d7442a4a Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Wed, 17 Mar 2021 17:49:44 -0700 Subject: [PATCH 10/25] apply suggestion --- piptools/scripts/sync.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 0996f1b09..f1f84346e 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -131,9 +131,7 @@ def cli( log.error(str(e)) sys.exit(2) - paths = None - if python_executable: - paths = get_sys_path_for_python_executable(python_executable) + paths = None if python_executable is None else get_sys_path_for_python_executable(python_executable) installed_dists = get_installed_distributions( skip=[], user_only=user_only, paths=paths, local_only=python_executable is None From 75a13c5f31d86f80ab2045df12a603a02c4379e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Mar 2021 00:50:07 +0000 Subject: [PATCH 11/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/scripts/sync.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index f1f84346e..0ae74447f 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -131,7 +131,11 @@ def cli( log.error(str(e)) sys.exit(2) - paths = None if python_executable is None else get_sys_path_for_python_executable(python_executable) + paths = ( + None + if python_executable is None + else get_sys_path_for_python_executable(python_executable) + ) installed_dists = get_installed_distributions( skip=[], user_only=user_only, paths=paths, local_only=python_executable is None From b3e9529d227d1d36a371f7f5316aa2255e297790 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Wed, 31 Mar 2021 17:43:21 -0700 Subject: [PATCH 12/25] add pip version check and support python executable resolving --- piptools/scripts/sync.py | 26 ++++++++++++++++++++++++-- piptools/utils.py | 10 ++++++++++ tests/test_cli_sync.py | 11 +++++++++-- tests/test_utils.py | 8 ++++++++ 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 0ae74447f..237d13976 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -1,6 +1,7 @@ import itertools import os import shlex +import shutil import sys from typing import List, Optional, Tuple, cast @@ -9,15 +10,21 @@ from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder from pip._internal.utils.misc import get_installed_distributions +from pip._vendor.packaging.version import Version from .. import sync from .._compat import parse_requirements from ..exceptions import PipToolsError from ..logging import log from ..repositories import PyPIRepository -from ..utils import flat_map, get_sys_path_for_python_executable +from ..utils import ( + flat_map, + get_pip_version_for_python_executable, + get_sys_path_for_python_executable, +) DEFAULT_REQUIREMENTS_FILE = "requirements.txt" +MIN_PIP_VERSION = Version("20.3") # Should be in sync with version in setup.cfg @click.command(context_settings={"help_option_names": ("-h", "--help")}) @@ -57,7 +64,6 @@ ) @click.option( "--python-executable", - type=click.Path(exists=True), help="Custom python executable path if targeting an environment other than current", ) @click.option("-v", "--verbose", count=True, help="Show more output") @@ -114,6 +120,22 @@ def cli( log.error("ERROR: " + msg) sys.exit(2) + if python_executable: + resolved_python_executable = shutil.which(python_executable) + if resolved_python_executable is None: + msg = "Could not resolve '{}' as valid executable path or alias" + log.error(msg.format(python_executable)) + sys.exit(2) + + pip_version = get_pip_version_for_python_executable(python_executable) + if pip_version < MIN_PIP_VERSION: + msg = ( + "Target python executable '{}' has pip version {} installed. " + "{} or higher is required." + ) + log.error(msg.format(python_executable, pip_version, MIN_PIP_VERSION)) + sys.exit(2) + install_command = cast(InstallCommand, create_command("install")) options, _ = install_command.parse_args([]) session = install_command._build_session(options) diff --git a/piptools/utils.py b/piptools/utils.py index 3fc3379a2..4cbdd5de8 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -312,6 +312,16 @@ def get_compile_command(click_ctx: click.Context) -> str: return " ".join(["pip-compile", *sorted(left_args), *sorted(right_args)]) +def get_pip_version_for_python_executable(python_executable: str) -> Version: + """ + Returns pip version for the given python executable. + """ + str_version = subprocess.check_output( # nosec + [python_executable, "-c", "import pip;print(pip.__version__)"], text=True + ) + return Version(str_version) + + def get_sys_path_for_python_executable(python_executable: str) -> List[str]: """ Returns sys.path list for the given python executable. diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index b78adb0c0..2ca26fdbe 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -286,11 +286,18 @@ def test_python_executable_option( assert called_install_options == [[custom_executable, "-m", "pip", "install", "-r"]] -def test_invalid_python_executable(runner): +@pytest.mark.parametrize( + "python_executable", + ( + ["/tmp/invalid_executable"], + ["invalid_python"], + ), +) +def test_invalid_python_executable(runner, python_executable): with open("requirements.txt", "w") as req_in: req_in.write("small-fake-a==1.10.0") - out = runner.invoke(cli, ["--python-executable", "/tmp/invalid_executable"]) + out = runner.invoke(cli, ["--python-executable", python_executable]) assert out.exit_code == 2, out diff --git a/tests/test_utils.py b/tests/test_utils.py index 50434a95c..bd5df27f7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,9 @@ import shlex import sys +import pip import pytest +from pip._vendor.packaging.version import Version from piptools.scripts.compile import cli as compile_cli from piptools.utils import ( @@ -15,6 +17,7 @@ format_specifier, get_compile_command, get_hashes_from_ireq, + get_pip_version_for_python_executable, get_sys_path_for_python_executable, is_pinned_requirement, is_url_requirement, @@ -371,6 +374,11 @@ def test_lookup_table_with_empty_values(): assert lookup_table((), operator.itemgetter(0)) == {} +def test_get_pip_version_for_python_executable(): + result = get_pip_version_for_python_executable(sys.executable) + assert Version(pip.__version__) == result + + def test_get_sys_path_for_python_executable(): result = get_sys_path_for_python_executable(sys.executable) # not testing for equality, because pytest adds extra paths into current sys.path From 92247e4243c2eb172922e42730c4d7a16e07d31f Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Wed, 31 Mar 2021 18:02:32 -0700 Subject: [PATCH 13/25] add test for invalid pip ver --- tests/test_cli_sync.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 2ca26fdbe..229a37bf8 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -1,8 +1,10 @@ +import os import subprocess import sys from unittest import mock import pytest +from pip._vendor.packaging.version import Version from piptools.scripts.sync import DEFAULT_REQUIREMENTS_FILE, cli @@ -244,6 +246,7 @@ def test_sync_dry_run_returns_non_zero_exit_code(runner): assert out.exit_code == 1 +@mock.patch("piptools.scripts.sync.get_pip_version_for_python_executable") @mock.patch("piptools.scripts.sync.get_sys_path_for_python_executable") @mock.patch("piptools.scripts.sync.get_installed_distributions") @mock.patch("piptools.sync.run") @@ -251,6 +254,7 @@ def test_python_executable_option( run, get_installed_distributions, get_sys_path_for_python_executable, + get_pip_version_for_python_executable, runner, fake_dist, ): @@ -260,13 +264,14 @@ def test_python_executable_option( with open("requirements.txt", "w") as req_in: req_in.write("small-fake-a==1.10.0") - custom_executable = "custom_executable" + custom_executable = os.path.abspath("custom_executable") with open(custom_executable, "w") as exec_file: exec_file.write("") sys_paths = ["", "./"] get_sys_path_for_python_executable.return_value = sys_paths get_installed_distributions.return_value = [fake_dist("django==1.8")] + get_pip_version_for_python_executable.return_value = Version("20.3") runner.invoke(cli, ["--python-executable", custom_executable]) @@ -289,8 +294,8 @@ def test_python_executable_option( @pytest.mark.parametrize( "python_executable", ( - ["/tmp/invalid_executable"], - ["invalid_python"], + "/tmp/invalid_executable", + "invalid_python", ), ) def test_invalid_python_executable(runner, python_executable): @@ -299,6 +304,30 @@ def test_invalid_python_executable(runner, python_executable): out = runner.invoke(cli, ["--python-executable", python_executable]) assert out.exit_code == 2, out + message = "Could not resolve '{}' as valid executable path or alias\n" + assert out.stderr == message.format(python_executable) + + +@mock.patch("piptools.scripts.sync.get_pip_version_for_python_executable") +def test_invalid_pip_version_in_python_executable( + get_pip_version_for_python_executable, runner +): + with open("requirements.txt", "w") as req_in: + req_in.write("small-fake-a==1.10.0") + + custom_executable = os.path.abspath("custom_executable") + with open(custom_executable, "w") as exec_file: + exec_file.write("") + + get_pip_version_for_python_executable.return_value = Version("19.1") + + out = runner.invoke(cli, ["--python-executable", custom_executable]) + assert out.exit_code == 2, out + message = ( + "Target python executable '{}' has pip version 19.1 installed. " + "20.3 or higher is required.\n" + ) + assert out.stderr == message.format(custom_executable) @mock.patch("piptools.sync.run") From d8750ba33d611340a8e8c3c57783b4b5c0b5ff1b Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Wed, 31 Mar 2021 19:18:30 -0700 Subject: [PATCH 14/25] fix tests for *nix --- piptools/utils.py | 2 +- tests/test_cli_sync.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index 4cbdd5de8..c4417e71c 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -317,7 +317,7 @@ def get_pip_version_for_python_executable(python_executable: str) -> Version: Returns pip version for the given python executable. """ str_version = subprocess.check_output( # nosec - [python_executable, "-c", "import pip;print(pip.__version__)"], text=True + [python_executable, "-c", "import pip;print(pip.__version__)"], encoding='utf8' ) return Version(str_version) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 229a37bf8..ad5e68054 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -268,6 +268,8 @@ def test_python_executable_option( with open(custom_executable, "w") as exec_file: exec_file.write("") + os.chmod(custom_executable, 0o700) + sys_paths = ["", "./"] get_sys_path_for_python_executable.return_value = sys_paths get_installed_distributions.return_value = [fake_dist("django==1.8")] @@ -319,6 +321,8 @@ def test_invalid_pip_version_in_python_executable( with open(custom_executable, "w") as exec_file: exec_file.write("") + os.chmod(custom_executable, 0o700) + get_pip_version_for_python_executable.return_value = Version("19.1") out = runner.invoke(cli, ["--python-executable", custom_executable]) From 7c7f15ba55df9f869b7cd835226b20828df2ff35 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Apr 2021 02:18:55 +0000 Subject: [PATCH 15/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- piptools/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index c4417e71c..7cfbd91ad 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -317,7 +317,7 @@ def get_pip_version_for_python_executable(python_executable: str) -> Version: Returns pip version for the given python executable. """ str_version = subprocess.check_output( # nosec - [python_executable, "-c", "import pip;print(pip.__version__)"], encoding='utf8' + [python_executable, "-c", "import pip;print(pip.__version__)"], encoding="utf8" ) return Version(str_version) From 17cc535e550ec4e4d28f579bc3c565c9fb31f1b9 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Mon, 10 May 2021 22:37:56 -0700 Subject: [PATCH 16/25] pr comments --- piptools/scripts/sync.py | 13 ++++++++----- piptools/subprocess_utils.py | 18 ++++++++++++++++++ piptools/utils.py | 22 +++++++++++++++++----- tests/test_cli_sync.py | 4 ++-- tests/test_subprocess_utils.py | 8 ++++++++ 5 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 piptools/subprocess_utils.py create mode 100644 tests/test_subprocess_utils.py diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 237d13976..385c560f5 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -10,7 +10,6 @@ from pip._internal.commands.install import InstallCommand from pip._internal.index.package_finder import PackageFinder from pip._internal.utils.misc import get_installed_distributions -from pip._vendor.packaging.version import Version from .. import sync from .._compat import parse_requirements @@ -20,11 +19,11 @@ from ..utils import ( flat_map, get_pip_version_for_python_executable, + get_required_pip_specification, get_sys_path_for_python_executable, ) DEFAULT_REQUIREMENTS_FILE = "requirements.txt" -MIN_PIP_VERSION = Version("20.3") # Should be in sync with version in setup.cfg @click.command(context_settings={"help_option_names": ("-h", "--help")}) @@ -127,13 +126,17 @@ def cli( log.error(msg.format(python_executable)) sys.exit(2) + # Ensure that target python executable has the right version of pip installed pip_version = get_pip_version_for_python_executable(python_executable) - if pip_version < MIN_PIP_VERSION: + required_pip_specification = get_required_pip_specification() + if not required_pip_specification.contains(pip_version): msg = ( "Target python executable '{}' has pip version {} installed. " - "{} or higher is required." + "Version {} is expected." + ) + log.error( + msg.format(python_executable, pip_version, required_pip_specification) ) - log.error(msg.format(python_executable, pip_version, MIN_PIP_VERSION)) sys.exit(2) install_command = cast(InstallCommand, create_command("install")) diff --git a/piptools/subprocess_utils.py b/piptools/subprocess_utils.py new file mode 100644 index 000000000..1421836a6 --- /dev/null +++ b/piptools/subprocess_utils.py @@ -0,0 +1,18 @@ +# WARNING! BE CAREFUL UPDATING THIS FILE +# Consider possible security implications associated with subprocess module. +import subprocess # nosec + + +def run_python_snippet(python_executable: str, code_to_run: str) -> str: + """ + Executes python code by calling python_executable with '-c' option. + """ + py_exec_cmd = python_executable, "-c", code_to_run + + # subprocess module should never be used with untrusted input + return subprocess.check_output( # nosec + py_exec_cmd, + shell=False, + text=True, + universal_newlines=True, + ) diff --git a/piptools/utils.py b/piptools/utils.py index 68155496f..dff64354b 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -3,7 +3,6 @@ import json import os import shlex -import subprocess # nosec from typing import ( Callable, Dict, @@ -26,6 +25,9 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.version import Version +from pip._vendor.pkg_resources import get_distribution + +from piptools.subprocess_utils import run_python_snippet _KT = TypeVar("_KT") _VT = TypeVar("_VT") @@ -363,12 +365,21 @@ def get_compile_command(click_ctx: click.Context) -> str: return " ".join(["pip-compile", *sorted(left_args), *sorted(right_args)]) +def get_required_pip_specification() -> SpecifierSet: + project_dist = get_distribution("pip-tools") + requirement = next((r for r in project_dist.requires() if r.name == "pip"), None) + assert ( + requirement is not None + ), "'pip' is expected to be in the list of pip-tools requirements" + return requirement.specifier + + def get_pip_version_for_python_executable(python_executable: str) -> Version: """ Returns pip version for the given python executable. """ - str_version = subprocess.check_output( # nosec - [python_executable, "-c", "import pip;print(pip.__version__)"], encoding="utf8" + str_version = run_python_snippet( + python_executable, "import pip;print(pip.__version__)" ) return Version(str_version) @@ -377,9 +388,10 @@ def get_sys_path_for_python_executable(python_executable: str) -> List[str]: """ Returns sys.path list for the given python executable. """ - result = subprocess.check_output( # nosec - [python_executable, "-c", "import sys;import json;print(json.dumps(sys.path))"] + result = run_python_snippet( + python_executable, "import sys;import json;print(json.dumps(sys.path))" ) + paths = json.loads(result) assert isinstance(paths, list) assert all(isinstance(i, str) for i in paths) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index ad5e68054..d265744e1 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -329,9 +329,9 @@ def test_invalid_pip_version_in_python_executable( assert out.exit_code == 2, out message = ( "Target python executable '{}' has pip version 19.1 installed. " - "20.3 or higher is required.\n" + "Version" # ">=20.3 is expected.\n" part is omitted ) - assert out.stderr == message.format(custom_executable) + assert out.stderr.startswith(message.format(custom_executable)) @mock.patch("piptools.sync.run") diff --git a/tests/test_subprocess_utils.py b/tests/test_subprocess_utils.py new file mode 100644 index 000000000..e2728e485 --- /dev/null +++ b/tests/test_subprocess_utils.py @@ -0,0 +1,8 @@ +import sys + +from piptools.subprocess_utils import run_python_snippet + + +def test_run_python_snippet_returns_multilne(): + result = run_python_snippet(sys.executable, r'print("MULTILINE\nOUTPUT", end="")') + assert result == "MULTILINE\nOUTPUT" From 36667bc8148ffe8d1013bb8dc6e2d53ba32c432c Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 8 Jun 2021 14:01:45 -0700 Subject: [PATCH 17/25] Extract _validate_python_executable, use logging formatting, PEP 257 docstrings --- piptools/scripts/sync.py | 43 ++++++++++++++++++++++------------------ piptools/utils.py | 3 +++ tests/test_cli_sync.py | 4 ++-- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 86974f794..5ab38929a 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -66,7 +66,7 @@ ) @click.option( "--python-executable", - help="Custom python executable path if targeting an environment other than current", + help="Custom python executable path if targeting an environment other than current.", ) @click.option("-v", "--verbose", count=True, help="Show more output") @click.option("-q", "--quiet", count=True, help="Give less output") @@ -123,24 +123,7 @@ def cli( sys.exit(2) if python_executable: - resolved_python_executable = shutil.which(python_executable) - if resolved_python_executable is None: - msg = "Could not resolve '{}' as valid executable path or alias" - log.error(msg.format(python_executable)) - sys.exit(2) - - # Ensure that target python executable has the right version of pip installed - pip_version = get_pip_version_for_python_executable(python_executable) - required_pip_specification = get_required_pip_specification() - if not required_pip_specification.contains(pip_version): - msg = ( - "Target python executable '{}' has pip version {} installed. " - "Version {} is expected." - ) - log.error( - msg.format(python_executable, pip_version, required_pip_specification) - ) - sys.exit(2) + _validate_python_executable(python_executable) install_command = cast(InstallCommand, create_command("install")) options, _ = install_command.parse_args([]) @@ -196,6 +179,28 @@ def cli( ) +def _validate_python_executable(python_executable: Optional[str]) -> None: + """ + Validates incoming python_executable argument passed to CLI. + """ + resolved_python_executable = shutil.which(python_executable) + if resolved_python_executable is None: + msg = "Could not resolve '%s' as valid executable path or alias." + log.error(msg, python_executable) + sys.exit(2) + + # Ensure that target python executable has the right version of pip installed + pip_version = get_pip_version_for_python_executable(python_executable) + required_pip_specification = get_required_pip_specification() + if not required_pip_specification.contains(pip_version): + msg = ( + "Target python executable '%s' has pip version %s installed. " + "Version %s is expected." + ) + log.error(msg, python_executable, pip_version, required_pip_specification) + sys.exit(2) + + def _compose_install_flags( finder: PackageFinder, no_index: bool, diff --git a/piptools/utils.py b/piptools/utils.py index dff64354b..23437b164 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -366,6 +366,9 @@ def get_compile_command(click_ctx: click.Context) -> str: def get_required_pip_specification() -> SpecifierSet: + """ + Returns pip version specifier requested by current pip-tools installation. + """ project_dist = get_distribution("pip-tools") requirement = next((r for r in project_dist.requires() if r.name == "pip"), None) assert ( diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index d265744e1..c0cce0acc 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -259,7 +259,7 @@ def test_python_executable_option( fake_dist, ): """ - Make sure sync command can run with `--python-executable` option + Make sure sync command can run with `--python-executable` option. """ with open("requirements.txt", "w") as req_in: req_in.write("small-fake-a==1.10.0") @@ -337,7 +337,7 @@ def test_invalid_pip_version_in_python_executable( @mock.patch("piptools.sync.run") def test_default_python_executable_option(run, runner): """ - Make sure sys.executable is used when --python-executable is not provided + Make sure sys.executable is used when --python-executable is not provided. """ with open("requirements.txt", "w") as req_in: req_in.write("small-fake-a==1.10.0") From 1d6e2a809f29a6ee2da9da2ba49b8be97d91a2d1 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 8 Jun 2021 14:05:13 -0700 Subject: [PATCH 18/25] fix mypy warning --- piptools/scripts/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index 5ab38929a..ce3082b93 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -179,7 +179,7 @@ def cli( ) -def _validate_python_executable(python_executable: Optional[str]) -> None: +def _validate_python_executable(python_executable: str) -> None: """ Validates incoming python_executable argument passed to CLI. """ From 24e214912905076210a6aca8d2c2cf5a2fc3b84a Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Tue, 8 Jun 2021 14:21:14 -0700 Subject: [PATCH 19/25] fix tests in python 3.6 --- piptools/subprocess_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piptools/subprocess_utils.py b/piptools/subprocess_utils.py index 1421836a6..42af78282 100644 --- a/piptools/subprocess_utils.py +++ b/piptools/subprocess_utils.py @@ -13,6 +13,5 @@ def run_python_snippet(python_executable: str, code_to_run: str) -> str: return subprocess.check_output( # nosec py_exec_cmd, shell=False, - text=True, universal_newlines=True, ) From 7319cc025a7e8627006b2ce1d4955f4ae80b7f9b Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Wed, 9 Jun 2021 12:13:57 -0700 Subject: [PATCH 20/25] revert log formatting --- piptools/scripts/sync.py | 12 +++++++----- tests/test_cli_sync.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index ce3082b93..5db6db834 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -185,8 +185,8 @@ def _validate_python_executable(python_executable: str) -> None: """ resolved_python_executable = shutil.which(python_executable) if resolved_python_executable is None: - msg = "Could not resolve '%s' as valid executable path or alias." - log.error(msg, python_executable) + msg = "Could not resolve '{}' as valid executable path or alias." + log.error(msg.format(python_executable)) sys.exit(2) # Ensure that target python executable has the right version of pip installed @@ -194,10 +194,12 @@ def _validate_python_executable(python_executable: str) -> None: required_pip_specification = get_required_pip_specification() if not required_pip_specification.contains(pip_version): msg = ( - "Target python executable '%s' has pip version %s installed. " - "Version %s is expected." + "Target python executable '{}' has pip version {} installed. " + "Version {} is expected." + ) + log.error( + msg.format(python_executable, pip_version, required_pip_specification) ) - log.error(msg, python_executable, pip_version, required_pip_specification) sys.exit(2) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index c0cce0acc..0214ee208 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -306,7 +306,7 @@ def test_invalid_python_executable(runner, python_executable): out = runner.invoke(cli, ["--python-executable", python_executable]) assert out.exit_code == 2, out - message = "Could not resolve '{}' as valid executable path or alias\n" + message = "Could not resolve '{}' as valid executable path or alias.\n" assert out.stderr == message.format(python_executable) From 5e8c59ddcf81a84984410476de08944aa100590b Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Thu, 10 Jun 2021 15:07:28 -0700 Subject: [PATCH 21/25] review comments --- piptools/scripts/sync.py | 2 +- piptools/utils.py | 4 +++- tests/test_cli_sync.py | 25 +++++-------------------- tests/test_utils.py | 1 + 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index c2c232005..10f4221c7 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -192,7 +192,7 @@ def _validate_python_executable(python_executable: str) -> None: # Ensure that target python executable has the right version of pip installed pip_version = get_pip_version_for_python_executable(python_executable) required_pip_specification = get_required_pip_specification() - if not required_pip_specification.contains(pip_version): + if not required_pip_specification.contains(pip_version, prereleases=True): msg = ( "Target python executable '{}' has pip version {} installed. " "Version {} is expected." diff --git a/piptools/utils.py b/piptools/utils.py index 23437b164..a05fb76a9 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -370,7 +370,9 @@ def get_required_pip_specification() -> SpecifierSet: Returns pip version specifier requested by current pip-tools installation. """ project_dist = get_distribution("pip-tools") - requirement = next((r for r in project_dist.requires() if r.name == "pip"), None) + requirement = next( + (r for r in project_dist.requires() if r.name == "pip"), None + ) # pragma: no cover assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 0214ee208..575b190fe 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -246,15 +246,9 @@ def test_sync_dry_run_returns_non_zero_exit_code(runner): assert out.exit_code == 1 -@mock.patch("piptools.scripts.sync.get_pip_version_for_python_executable") -@mock.patch("piptools.scripts.sync.get_sys_path_for_python_executable") -@mock.patch("piptools.scripts.sync.get_installed_distributions") @mock.patch("piptools.sync.run") def test_python_executable_option( run, - get_installed_distributions, - get_sys_path_for_python_executable, - get_pip_version_for_python_executable, runner, fake_dist, ): @@ -264,31 +258,22 @@ def test_python_executable_option( with open("requirements.txt", "w") as req_in: req_in.write("small-fake-a==1.10.0") - custom_executable = os.path.abspath("custom_executable") - with open(custom_executable, "w") as exec_file: - exec_file.write("") + custom_executable = os.path.abspath(sys.executable) os.chmod(custom_executable, 0o700) - sys_paths = ["", "./"] - get_sys_path_for_python_executable.return_value = sys_paths - get_installed_distributions.return_value = [fake_dist("django==1.8")] - get_pip_version_for_python_executable.return_value = Version("20.3") - runner.invoke(cli, ["--python-executable", custom_executable]) - get_installed_distributions.assert_called_once_with( - skip=[], user_only=False, paths=sys_paths, local_only=False - ) - assert run.call_count == 2 call_args = [call[0][0] for call in run.call_args_list] - called_uninstall_options = [args for args in call_args if args[3] == "uninstall"] + called_uninstall_options = [ + args[:5] for args in call_args if args[3] == "uninstall" + ] called_install_options = [args[:-1] for args in call_args if args[3] == "install"] assert called_uninstall_options == [ - [custom_executable, "-m", "pip", "uninstall", "-y", "django"] + [custom_executable, "-m", "pip", "uninstall", "-y"] ] assert called_install_options == [[custom_executable, "-m", "pip", "install", "-r"]] diff --git a/tests/test_utils.py b/tests/test_utils.py index d9179ee8d..6a07d585b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -437,6 +437,7 @@ def test_get_pip_version_for_python_executable(): def test_get_sys_path_for_python_executable(): result = get_sys_path_for_python_executable(sys.executable) + assert result, "get_sys_path_for_python_executable should not return empty result" # not testing for equality, because pytest adds extra paths into current sys.path for path in result: assert path in sys.path From cd26a3a537493f780c0eef082eb2ef081388082a Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Thu, 10 Jun 2021 21:28:15 -0700 Subject: [PATCH 22/25] Update piptools/utils.py Co-authored-by: Albert Tugushev --- piptools/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/utils.py b/piptools/utils.py index a05fb76a9..d6ffd8d1b 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -372,7 +372,7 @@ def get_required_pip_specification() -> SpecifierSet: project_dist = get_distribution("pip-tools") requirement = next( (r for r in project_dist.requires() if r.name == "pip"), None - ) # pragma: no cover + ) # pragma: no branch assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" From 390a150c2676cf3524d4cf73743d901727f73554 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Thu, 10 Jun 2021 23:01:10 -0700 Subject: [PATCH 23/25] try no cover --- piptools/utils.py | 2 +- tests/test_cli_sync.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index d6ffd8d1b..a05fb76a9 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -372,7 +372,7 @@ def get_required_pip_specification() -> SpecifierSet: project_dist = get_distribution("pip-tools") requirement = next( (r for r in project_dist.requires() if r.name == "pip"), None - ) # pragma: no branch + ) # pragma: no cover assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 575b190fe..e1e222067 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -306,8 +306,6 @@ def test_invalid_pip_version_in_python_executable( with open(custom_executable, "w") as exec_file: exec_file.write("") - os.chmod(custom_executable, 0o700) - get_pip_version_for_python_executable.return_value = Version("19.1") out = runner.invoke(cli, ["--python-executable", custom_executable]) From 13eb866b7fa420b0dc8615d687eda413a3fe9e53 Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Fri, 11 Jun 2021 21:09:34 -0700 Subject: [PATCH 24/25] Update piptools/utils.py Co-authored-by: Albert Tugushev --- piptools/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index a05fb76a9..280614d00 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -370,9 +370,9 @@ def get_required_pip_specification() -> SpecifierSet: Returns pip version specifier requested by current pip-tools installation. """ project_dist = get_distribution("pip-tools") - requirement = next( + requirement = next( # pragma: no branch (r for r in project_dist.requires() if r.name == "pip"), None - ) # pragma: no cover + ) assert ( requirement is not None ), "'pip' is expected to be in the list of pip-tools requirements" From a138d078bcfa8c486ad987553ca7eb6c180aaf0b Mon Sep 17 00:00:00 2001 From: Marat Khabibullin Date: Sat, 12 Jun 2021 10:50:12 -0700 Subject: [PATCH 25/25] fix tests --- tests/test_cli_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index e1e222067..aabfa7c7a 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -260,8 +260,6 @@ def test_python_executable_option( custom_executable = os.path.abspath(sys.executable) - os.chmod(custom_executable, 0o700) - runner.invoke(cli, ["--python-executable", custom_executable]) assert run.call_count == 2 @@ -306,6 +304,8 @@ def test_invalid_pip_version_in_python_executable( with open(custom_executable, "w") as exec_file: exec_file.write("") + os.chmod(custom_executable, 0o700) + get_pip_version_for_python_executable.return_value = Version("19.1") out = runner.invoke(cli, ["--python-executable", custom_executable])