From 9fa94256c8236582a971f2e07d89ff7bb02ab014 Mon Sep 17 00:00:00 2001 From: david <14880945+ddelange@users.noreply.github.com> Date: Sun, 26 Jan 2020 22:39:56 +0100 Subject: [PATCH] :sparkles: Add --install flag (#11) * :sparkles: Add --install flag * :bug: Remove **kwargs as it's invalid syntax on py27 * :bug: Increase stability for --install -e . * :green_heart: Run CD only when tags are pushed to master * :pencil: Remove useless sh syntax highlighting * :loud_sound: Add environment info in -vvv * :art: Clean up .pre-commit-config.yaml --- .github/workflows/main.yml | 1 + .pre-commit-config.yaml | 30 ++------ README.md | 141 ++++++++++++++++++++++++------------- src/pipgrip/cli.py | 50 ++++++++++++- src/pipgrip/pipper.py | 93 ++++++++++++++++++++++-- tests/test_cli.py | 3 +- 6 files changed, 235 insertions(+), 83 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f521ca5..36f7ff4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,6 +4,7 @@ on: pull_request: branches: "*" push: + branches: "master" tags: "*" jobs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8bb4e1f..dc01b1e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,21 +12,10 @@ repos: - id: black language_version: python3.7 -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: - - id: trailing-whitespace - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: - - id: end-of-file-fixer - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 +- repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.21 hooks: - - id: mixed-line-ending - args: ['--fix=lf'] + - id: isort - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 @@ -44,13 +33,8 @@ repos: flake8-quotes, flake8-tuple, ] - -- repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.21 - hooks: - - id: isort - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: + - id: mixed-line-ending + args: ['--fix=lf'] + - id: trailing-whitespace + - id: end-of-file-fixer - id: check-merge-conflict diff --git a/README.md b/README.md index 1ed10ac..7813c8a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # pipgrip -[![build](https://img.shields.io/github/workflow/status/ddelange/pipgrip/GH/master?logo=github&cacheSeconds=86400)](https://github.com/ddelange/pipgrip/actions) +[![build](https://img.shields.io/github/workflow/status/ddelange/pipgrip/GH/master?logo=github&cacheSeconds=86400)](https://github.com/ddelange/pipgrip/actions?query=branch%3Amaster) [![codecov](https://img.shields.io/codecov/c/github/ddelange/pipgrip/master?logo=codecov&logoColor=white)](https://codecov.io/gh/ddelange/pipgrip) [![pypi Version](https://img.shields.io/pypi/v/pipgrip.svg?logo=pypi&logoColor=white)](https://pypi.org/project/pipgrip/) -[![python](https://img.shields.io/pypi/pyversions/pipgrip.svg?logo=python&logoColor=white)](https://github.com/ddelange/pipgrip/releases/latest) +[![python](https://img.shields.io/pypi/pyversions/pipgrip.svg?logo=python&logoColor=white)](https://pypi.org/project/pipgrip/) [![downloads](https://pepy.tech/badge/pipgrip)](https://pypistats.org/packages/pipgrip) [![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) @@ -24,11 +24,12 @@ For offline usage, [pipdeptree](https://github.com/naiquevin/pipdeptree) can inspect the current environment and show how the currently installed packages relate to each other. This however requires the packages to be pip-installed, and (despite warnings about e.g. cyclic dependencies) offers no form of dependency resolution since it's only based on the (single) package versions installed in the environment. Such shortcomings are avoided when using pipgrip, since packages don't need to be installed and all versions available to pip are considered. + ## Installation This pure-Python, OS independent package is available on [PyPI](https://pypi.org/project/pipgrip/): -```sh +``` pip install pipgrip ``` @@ -36,24 +37,32 @@ pip install pipgrip ## Usage This package can be used to: -- Alleviate [Python dependency hell](https://medium.com/knerd/the-nine-circles-of-python-dependency-hell-481d53e3e025) by resolving the latest viable combination of required packages -- Render an exhaustive dependency tree for any given pip-compatible package(s) with `--tree` -- Detect version conflicts for given constraints and give human readable feedback about it -- Find dependency conflicts or cyclic dependencies in local projects: - - `pipgrip -v --tree .` -- Avoid bugs by running pipgrip as a stage in CI pipelines -- Install complex packages without worries using: +- **Render** an exhaustive dependency tree for any given pip-compatible package(s) with `--tree` +- **Alleviate** [Python dependency hell](https://medium.com/knerd/the-nine-circles-of-python-dependency-hell-481d53e3e025) by resolving the latest viable combination of required packages +- **Avoid** bugs by running pipgrip as a stage in CI pipelines +- **Detect** version conflicts for given constraints and give human readable feedback about it +- **Warn** for cyclic dependencies in local projects [and install them anyway]: + - `pipgrip -v --tree . [--install -e]` +- **Install** complex packages without worries using: + - `pipgrip --install aiobotocore[awscli]` - ``pip install -U --no-deps `pipgrip --pipe aiobotocore[awscli]` `` -- Generate a lockfile with a complete working set of dependencies for worriless installs (see [known caveats](#known-caveats)): +- **Generate** a lockfile with a complete working set of dependencies for worriless installs: + - `pipgrip --lock --install --tree -v aiobotocore[awscli]` - `pipgrip --lock -tree aiobotocore[awscli] && pip install -U --no-deps -r ./pipgrip.lock` - `pipgrip aiobotocore[awscli] | pip install -U --no-deps -r /dev/stdin` +- **Combine** dependency trees of multiple packages into one unified set of pinned packages: + - `pipgrip --lock --install --tree -v .[boto3] s3transfer==0.2.1` + +See also [known caveats](#known-caveats). -```sh +``` $ pipgrip --help Usage: pipgrip [OPTIONS] [DEPENDENCIES]... Options: + --install Install full dependency tree after resolving. + -e, --editable Install a project in editable mode. --lock Write out pins to './pipgrip.lock'. --pipe Output space-separated pins instead of newline- separated pins. @@ -80,41 +89,67 @@ Options: #### Dependency trees Exhaustive dependency trees without the need to install any packages (at most build some wheels). -```sh +``` $ pipgrip --tree pipgrip -pipgrip (0.0.3) -├── anytree (2.7.3) -│ └── six (1.13.0) +pipgrip (0.1.0) +├── anytree (2.8.0) +│ └── six>=1.9.0 (1.14.0) ├── click (7.0) -├── packaging>=17 (20.0) +├── packaging>=17 (20.1) │ ├── pyparsing>=2.0.2 (2.4.6) -│ └── six (1.13.0) +│ └── six (1.14.0) ├── pkginfo>=1.4.2 (1.5.0.1) -├── setuptools>=38.3 (44.0.0) +├── setuptools>=38.3 (45.1.0) └── wheel (0.33.6) ``` #### Lockfile generation Using the `--lock` option, resolved (pinned) dependencies are additionally written to `./pipgrip.lock`. -```sh -$ pipgrip --lock boto3 - -boto3==1.10.46 -botocore==1.13.46 +``` +$ pipgrip --tree --lock botocore==1.13.48 'boto3>=1.10' + +botocore==1.13.48 (1.13.48) +├── docutils<0.16,>=0.10 (0.15.2) +├── jmespath<1.0.0,>=0.7.1 (0.9.4) +├── python-dateutil<3.0.0,>=2.1 (2.8.1) +│ └── six>=1.5 (1.14.0) +└── urllib3<1.26,>=1.20 (1.25.8) +boto3 (1.10.48) +├── botocore<1.14.0,>=1.13.48 (1.13.48) +│ ├── docutils<0.16,>=0.10 (0.15.2) +│ ├── jmespath<1.0.0,>=0.7.1 (0.9.4) +│ ├── python-dateutil<3.0.0,>=2.1 (2.8.1) +│ │ └── six>=1.5 (1.14.0) +│ └── urllib3<1.26,>=1.20 (1.25.8) +├── jmespath<1.0.0,>=0.7.1 (0.9.4) +└── s3transfer<0.3.0,>=0.2.0 (0.2.1) + └── botocore<2.0.0,>=1.12.36 (1.13.48) + ├── docutils<0.16,>=0.10 (0.15.2) + ├── jmespath<1.0.0,>=0.7.1 (0.9.4) + ├── python-dateutil<3.0.0,>=2.1 (2.8.1) + │ └── six>=1.5 (1.14.0) + └── urllib3<1.26,>=1.20 (1.25.8) + +$ cat ./pipgrip.lock + +botocore==1.13.48 docutils==0.15.2 jmespath==0.9.4 python-dateutil==2.8.1 -six==1.13.0 -urllib3==1.25.7 +six==1.14.0 +urllib3==1.25.8 +boto3==1.10.48 s3transfer==0.2.1 ``` +NOTE: +Since the selected botocore version is older than the one required by the recent versions of boto3, all boto3 versions will be checked for compatibility with botocore==1.12.42. #### Version conflicts If version conflicts exist for the given (ranges of) package version(s), a verbose explanation is raised. -```sh +``` $ pipgrip auto-sklearn~=0.6 dragnet==2.0.4 Error: Because dragnet (2.0.4) depends on scikit-learn (>=0.15.2,<0.21.0) @@ -128,46 +163,53 @@ If older versions of auto-sklearn are allowed, PubGrub will try all acceptable v #### Cyclic dependencies If cyclic dependencies are found, it is noted in the resulting tree. -```sh -$ pipgrip --tree keras==2.2.2 +``` +$ pipgrip --tree -v keras==2.2.2 +WARNING: Cyclic dependency found: keras depends on keras-applications and vice versa. +WARNING: Cyclic dependency found: keras depends on keras-preprocessing and vice versa. keras==2.2.2 (2.2.2) ├── h5py (2.10.0) -│ ├── numpy>=1.9.1 (1.18.0) -│ └── six>=1.9.0 (1.13.0) +│ ├── numpy>=1.7 (1.18.1) +│ └── six (1.14.0) ├── keras-applications==1.0.4 (1.0.4) │ ├── h5py (2.10.0) -│ │ ├── numpy>=1.9.1 (1.18.0) -│ │ └── six>=1.9.0 (1.13.0) -│ ├── keras==2.2.2 (2.2.2, cyclic) -│ └── numpy>=1.9.1 (1.18.0) +│ │ ├── numpy>=1.7 (1.18.1) +│ │ └── six (1.14.0) +│ ├── keras>=2.1.6 (2.2.2, cyclic) +│ └── numpy>=1.9.1 (1.18.1) ├── keras-preprocessing==1.0.2 (1.0.2) -│ ├── keras==2.2.2 (2.2.2, cyclic) -│ ├── numpy>=1.9.1 (1.18.0) +│ ├── keras>=2.1.6 (2.2.2, cyclic) +│ ├── numpy>=1.9.1 (1.18.1) │ ├── scipy>=0.14 (1.4.1) -│ │ └── numpy>=1.9.1 (1.18.0) -│ └── six>=1.9.0 (1.13.0) -├── numpy>=1.9.1 (1.18.0) -├── pyyaml (5.2) +│ │ └── numpy>=1.13.3 (1.18.1) +│ └── six>=1.9.0 (1.14.0) +├── numpy>=1.9.1 (1.18.1) +├── pyyaml (5.3) ├── scipy>=0.14 (1.4.1) -│ └── numpy>=1.9.1 (1.18.0) -└── six>=1.9.0 (1.13.0) +│ └── numpy>=1.13.3 (1.18.1) +└── six>=1.9.0 (1.14.0) ``` + ## Known caveats -- ``pip install -U `pipgrip package` `` without `--no-deps` is unsafe while pip doesn't [yet](https://twitter.com/di_codes/status/1193980331004743680) have a built-in dependency resolver, and leaves room for interpretation by pip +- ``pip install -U `pipgrip --pipe package` `` without `--no-deps` is unsafe while pip doesn't [yet](https://twitter.com/di_codes/status/1193980331004743680) have a built-in dependency resolver, and leaves room for interpretation by pip - Package names are canonicalised in wheel metadata, resulting in e.g. `path.py -> path-py` and `keras_preprocessing -> keras-preprocessing` in output - [VCS Support](https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support) isn't implemented yet - `--reversed-tree` isn't implemented yet -- Since `pip install -r` does not accept `.` as requirement, it is omitted from lockfiles, so `--pipe` should be used when installing local projects -- Installing packages using pipgrip is not very intuitive, so maybe pipgrip needs a stable `--install` flag +- Since `pip install -r` does not accept `.` as requirement, it is omitted from lockfiles, so `--install` or `--pipe` should be used when installing local projects +- The equivalent of e.g. `pip install ../aiobotocore[boto3]` is not yet implemented. However, e.g. `pipgrip --install .[boto3]` is allowed. + ## Development +[![gitmoji](https://img.shields.io/badge/gitmoji-%20%F0%9F%98%9C%20%F0%9F%98%8D-ffdd67)](https://github.com/carloscuesta/gitmoji-cli) +[![pre-commit](https://img.shields.io/badge/pre--commit-available-green?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) + Create a virtual environment and get ready to develop: -```sh +``` make install ``` @@ -175,7 +217,7 @@ This [make-command](Makefile) is equivalent to the following steps: Install pre-commit and other continous integration dependencies in order to make commits and run tests. -```sh +``` pip install -r requirements/ci.txt pre-commit install ``` @@ -184,10 +226,11 @@ With requirements installed, `make lint` and `make test` can now be run. There i To import the package in the python environment, install the package (`-e` for editable installation, upon import, python will read directly from the repository). -```sh +``` pip install -e . ``` + ## See also - [PubGrub spec](https://github.com/dart-lang/pub/blob/SDK-2.2.1-dev.3.0/doc/solver.md) diff --git a/src/pipgrip/cli.py b/src/pipgrip/cli.py index 39ce28d..201ada9 100755 --- a/src/pipgrip/cli.py +++ b/src/pipgrip/cli.py @@ -9,14 +9,16 @@ import click from anytree import Node, RenderTree from anytree.search import findall_by_attr +from packaging.markers import default_environment from pipgrip.compat import USER_CACHE_DIR from pipgrip.libs.mixology.failure import SolverFailure from pipgrip.libs.mixology.package import Package from pipgrip.libs.mixology.version_solver import VersionSolver from pipgrip.package_source import PackageSource +from pipgrip.pipper import install_packages -logging.basicConfig(format="%(asctime)s:%(levelname)s:%(name)s:%(message)s") +logging.basicConfig(format="%(levelname)s: %(message)s") logger = logging.getLogger() logger.setLevel(logging.ERROR) @@ -96,7 +98,7 @@ def _recurse_dependencies( def build_tree(source, decision_packages): - tree_root = Node("root") + tree_root = Node("__root__") exhaustive = _recurse_dependencies( source, decision_packages, source._root_dependencies, tree_root, tree_root ) @@ -124,6 +126,12 @@ def render_tree(root_tree, max_depth): @click.command() @click.argument("dependencies", nargs=-1) +@click.option( + "--install", is_flag=True, help="Install full dependency tree after resolving.", +) +@click.option( + "-e", "--editable", is_flag=True, help="Install a project in editable mode.", +) @click.option( "--lock", is_flag=True, help="Write out pins to './pipgrip.lock'.", ) @@ -189,6 +197,8 @@ def render_tree(root_tree, max_depth): ) def main( dependencies, + install, + editable, lock, pipe, json, @@ -208,6 +218,7 @@ def main( logger.setLevel(logging.INFO) if verbose >= 3: logger.setLevel(logging.DEBUG) + logger.debug(str(default_environment())) if sum((pipe, json, tree, reversed_tree)) > 1: raise click.ClickException("Illegal combination of output formats selected") @@ -219,6 +230,20 @@ def main( raise click.ClickException( "--max-depth has no effect without --tree or --reversed-tree" ) + if editable: + if not install: + raise click.ClickException("--editable has no effect without --install") + if len(dependencies) > 1 or not dependencies[0].startswith("."): + raise click.ClickException( + "--editable does not accept input '{}'".format(" ".join(dependencies)) + ) + for dep in dependencies: + if os.sep in dep: + raise click.ClickException( + "'{}' looks like a path, and is not supported yet by pipgrip".format( + dep + ) + ) if reversed_tree: tree = True @@ -274,8 +299,27 @@ def main( elif json: output = dumps(packages) else: - output = "\n".join(["==".join(x) for x in packages.items()]) + output = "\n".join( + [ + "==".join(x) if not x[0].startswith(".") else x[0] + for x in packages.items() + ] + ) click.echo(output) + + if install: + install_packages( + # sort to ensure . is added right after --editable + sorted( + "==".join(x) if not x[0].startswith(".") else x[0] + for x in packages.items() + ), + index_url, + extra_index_url, + pre, + cache_dir, + editable, + ) except (SolverFailure, click.ClickException, CalledProcessError) as exc: raise click.ClickException(str(exc)) # except Exception as exc: diff --git a/src/pipgrip/pipper.py b/src/pipgrip/pipper.py index 899ff73..908b5e2 100644 --- a/src/pipgrip/pipper.py +++ b/src/pipgrip/pipper.py @@ -5,6 +5,7 @@ import sys import pkg_resources +from click import echo as _echo from packaging.markers import default_environment from packaging.utils import canonicalize_name from pkginfo import get_metadata @@ -27,6 +28,7 @@ def parse_req(requirement, extras=None): req.extras = extras req.key = "." full_str = req.__str__().replace(req.name, req.key) + req.name = req.key else: req = pkg_resources.Requirement.parse(requirement) if extras is not None: @@ -46,6 +48,70 @@ def __str__(): return req +def stream_bash_command(bash_command, echo=False): + # https://gist.github.com/jaketame/3ed43d1c52e9abccd742b1792c449079 + # https://gist.github.com/bgreenlee/1402841 + process = subprocess.Popen( + bash_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + + def check_io(): + output = "" + while True: + line = process.stdout.readline().decode("utf-8") + if line: + output += line + if echo: + _echo(line[:-1]) + else: + break + return output + + # keep checking stdout/stderr until the child exits + out = "" + while process.poll() is None: + out += check_io() + + return_code = process.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, bash_command, output=out) + + return out + + +def _get_install_args(index_url, extra_index_url, pre, cache_dir, editable): + args = [ + sys.executable, + "-m", + "pip", + "install", + "-U", + "--no-deps", + "--cache-dir", + cache_dir, + ] + if index_url is not None: + args += [ + "--index-url", + index_url, + "--trusted-host", + urlparse(index_url).hostname, + ] + if extra_index_url is not None: + args += [ + "--extra-index-url", + extra_index_url, + "--trusted-host", + urlparse(extra_index_url).hostname, + ] + if PIP_VERSION >= [10]: + args.append("--progress-bar=off") + if editable: + args += ["--editable"] + + return args + + def _get_wheel_args(index_url, extra_index_url, pre, cache_dir=None): args = [ sys.executable, @@ -81,19 +147,34 @@ def _get_wheel_args(index_url, extra_index_url, pre, cache_dir=None): return args +def install_packages(packages, index_url, extra_index_url, pre, cache_dir, editable): + """Install a list of packages with pip.""" + args = ( + _get_install_args(index_url, extra_index_url, pre, cache_dir, editable) + + packages + ) + try: + out = stream_bash_command(args, echo=True) + except subprocess.CalledProcessError as err: + output = getattr(err, "output") or "" + logger.error(output) + raise + return out + + def _get_available_versions(package, index_url, extra_index_url, pre): logger.debug("Finding possible versions for {}".format(package)) args = _get_wheel_args(index_url, extra_index_url, pre) + [package + "==rubbish"] try: - out = subprocess.check_output(args, stderr=subprocess.STDOUT) + out = stream_bash_command(args) except subprocess.CalledProcessError as err: # expected. we forced this by using a non-existing version number. - out = getattr(err, "output", b"") + out = getattr(err, "output") or "" else: logger.warning(out) raise RuntimeError("Unexpected success:" + " ".join(args)) - out = out.decode("utf-8").splitlines() + out = out.splitlines() for line in out[::-1]: if "Could not find a version that satisfies the requirement" in line: all_versions = line.split("from versions: ", 1)[1].rstrip(")").split(", ") @@ -119,12 +200,12 @@ def _download_wheel(package, index_url, extra_index_url, pre, cache_dir): abs_cache_dir = os.path.abspath(os.path.expanduser(cache_dir)) args = _get_wheel_args(index_url, extra_index_url, pre, cache_dir) + [package] try: - out = subprocess.check_output(args, stderr=subprocess.STDOUT,) + out = stream_bash_command(args) except subprocess.CalledProcessError as err: - output = getattr(err, "output", b"").decode("utf-8") + output = getattr(err, "output") or "" logger.error(output) raise - out = out.decode("utf-8").splitlines()[::-1] + out = out.splitlines()[::-1] for i, line in enumerate(out): if cache_dir in line or abs_cache_dir in line or "Stored in directory" in line: if "Stored in directory" in line: diff --git a/tests/test_cli.py b/tests/test_cli.py index 3a61a7e..cd4076d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,6 @@ from click.testing import CliRunner import pipgrip.pipper -from pipgrip import __version__ from pipgrip.cli import flatten, main from pipgrip.pipper import _download_wheel @@ -73,7 +72,7 @@ def mock_get_available_versions(package, *args, **kwargs): ( ["."], [ - ".==" + __version__, + ".", "anytree==2.7.3", "six==1.13.0", "click==7.0",