diff --git a/docs/docs.md b/docs/docs.md index 5e078bdff7..daad583b68 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -206,25 +206,13 @@ optional arguments: ``` pipx upgrade-all --help -usage: pipx upgrade-all [-h] [--include-deps] [--system-site-packages] - [--index-url INDEX_URL] [--editable] - [--pip-args PIP_ARGS] [--skip SKIP [SKIP ...]] - [--force] [--verbose] +usage: pipx upgrade-all [-h] [--skip SKIP [SKIP ...]] [--force] [--verbose] Upgrades all packages within their virtual environments by running 'pip install --upgrade PACKAGE' optional arguments: -h, --help show this help message and exit - --include-deps Include apps of dependent packages - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --index-url INDEX_URL, -i INDEX_URL - Base URL of Python Package Index - --editable, -e Install a project in editable mode - --pip-args PIP_ARGS Arbitrary pip arguments to pass directly to pip - install/upgrade commands --skip SKIP [SKIP ...] skip these packages --force, -f Modify existing virtual environment and files in @@ -307,35 +295,20 @@ optional arguments: ``` pipx reinstall-all --help -usage: pipx reinstall-all [-h] [--include-deps] [--system-site-packages] - [--index-url INDEX_URL] [--editable] - [--pip-args PIP_ARGS] [--skip SKIP [SKIP ...]] - [--verbose] - python +usage: pipx reinstall-all [-h] [--skip SKIP [SKIP ...]] [--verbose] python Reinstalls all packages using a different version of Python. -Packages are uninstalled, then installed with pipx install PACKAGE. +Packages are uninstalled, then installed with pipx install PACKAGE +with the same options used in the original install of PACKAGE. This is useful if you upgraded to a new version of Python and want all your packages to use the latest as well. -If you originally installed a package from a source other than PyPI, -this command may behave in unexpected ways since it will reinstall from PyPI. - positional arguments: python optional arguments: -h, --help show this help message and exit - --include-deps Include apps of dependent packages - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --index-url INDEX_URL, -i INDEX_URL - Base URL of Python Package Index - --editable, -e Install a project in editable mode - --pip-args PIP_ARGS Arbitrary pip arguments to pass directly to pip - install/upgrade commands --skip SKIP [SKIP ...] skip these packages --verbose diff --git a/src/pipx/commands/commands.py b/src/pipx/commands/commands.py index e36ef6b099..68be49d8b7 100644 --- a/src/pipx/commands/commands.py +++ b/src/pipx/commands/commands.py @@ -41,7 +41,7 @@ rmdir, run_pypackage_bin, ) -from pipx.venv import Venv, VenvContainer +from pipx.venv import Venv, VenvContainer, PackageInstallFailureError def run( @@ -126,7 +126,7 @@ def run( def _download_and_run( venv_dir: Path, - package: str, + package_or_url: str, app: str, binary_args: List[str], python: str, @@ -136,12 +136,27 @@ def _download_and_run( ): venv = Venv(venv_dir, python=python, verbose=verbose) venv.create_venv(venv_args, pip_args) - venv.install_package(package, pip_args) + + # venv.pipx_metadata.main_package.package contains package name if it is + # pre-existing, otherwise is None to instruct venv.install_package to + # determine package name. + + try: + venv.install_package( + package=venv.pipx_metadata.main_package.package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=False, + include_apps=True, + is_main_package=True, + ) + except PackageInstallFailureError: + raise PipxError(f"Unable to install {package_or_url}") if not (venv.bin_path / app).exists(): - apps = venv.get_venv_metadata_for_package(package).apps + apps = venv.pipx_metadata.main_package.apps raise PipxError( - f"'{app}' executable script not found in package '{package}'. " + f"'{app}' executable script not found in package '{package_or_url}'. " "Available executable scripts: " f"{', '.join(b for b in apps)}" ) @@ -229,11 +244,20 @@ def install( venv = Venv(venv_dir, python=python, verbose=verbose) try: venv.create_venv(venv_args, pip_args) - venv.install_package(package_or_url, pip_args) - - if venv.get_venv_metadata_for_package(package).package_version is None: + try: + venv.install_package( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=True, + is_main_package=True, + ) + except PackageInstallFailureError: venv.remove_venv() - raise PipxError(f"Could not find package {package}. Is the name correct?") + raise PipxError( + f"Could not install package {package}. Is the name or spec correct?" + ) _run_post_install_actions( venv, package, local_bin_dir, venv_dir, include_dependencies, force=force @@ -253,12 +277,12 @@ def _run_post_install_actions( *, force: bool, ): - metadata = venv.get_venv_metadata_for_package(package) + package_metadata = venv.package_metadata[package] - if not metadata.app_paths and not include_dependencies: + if not package_metadata.app_paths and not include_dependencies: # No apps associated with this package and we aren't including dependencies. # This package has nothing for pipx to use, so this is an error. - for dep, dependent_apps in metadata.app_paths_of_dependencies.items(): + for dep, dependent_apps in package_metadata.app_paths_of_dependencies.items(): print( f"Note: Dependent package '{dep}' contains {len(dependent_apps)} apps" ) @@ -268,7 +292,7 @@ def _run_post_install_actions( if venv.safe_to_remove(): venv.remove_venv() - if len(metadata.app_paths_of_dependencies.keys()): + if len(package_metadata.app_paths_of_dependencies.keys()): raise PipxError( f"No apps associated with package {package}. " "Try again with '--include-deps' to include apps of dependent packages, " @@ -283,9 +307,9 @@ def _run_post_install_actions( "Consider using pip or a similar tool instead." ) - if metadata.apps: + if package_metadata.apps: pass - elif metadata.apps_of_dependencies and include_dependencies: + elif package_metadata.apps_of_dependencies and include_dependencies: pass else: # No apps associated with this package and we aren't including dependencies. @@ -298,10 +322,12 @@ def _run_post_install_actions( "Consider using pip or a similar tool instead." ) - _expose_apps_globally(local_bin_dir, metadata.app_paths, package, force=force) + _expose_apps_globally( + local_bin_dir, package_metadata.app_paths, package, force=force + ) if include_dependencies: - for _, app_paths in metadata.app_paths_of_dependencies.items(): + for _, app_paths in package_metadata.app_paths_of_dependencies.items(): _expose_apps_globally(local_bin_dir, app_paths, package, force=force) print(_get_package_summary(venv_dir, package=package, new_install=True)) @@ -323,6 +349,7 @@ def _warn_if_not_on_path(local_bin_dir: Path): def inject( venv_dir: Path, package: str, + package_or_url: str, pip_args: List[str], *, verbose: bool, @@ -341,8 +368,19 @@ def inject( ) venv = Venv(venv_dir, verbose=verbose) - venv.install_package(package, pip_args) - + try: + venv.install_package( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=include_apps, + is_main_package=False, + ) + except PackageInstallFailureError: + raise PipxError( + f"Could not inject package {package}. Is the name or spec correct?" + ) if include_apps: _run_post_install_actions( venv, @@ -358,6 +396,9 @@ def inject( def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): + """Uninstall entire venv_dir, including main package and all injected + packages. + """ if not venv_dir.exists(): print(f"Nothing to uninstall for {package} 😴") app = which(package) @@ -369,27 +410,35 @@ def uninstall(venv_dir: Path, package: str, local_bin_dir: Path, verbose: bool): venv = Venv(venv_dir, verbose=verbose) - if venv.python_path.is_file(): - # has a valid python interpreter and can get metadata about the package - metadata = venv.get_venv_metadata_for_package(package) - app_paths = metadata.app_paths - for dep_paths in metadata.app_paths_of_dependencies.values(): - app_paths += dep_paths + if venv.pipx_metadata.main_package is not None: + app_paths: List[Path] = [] + for viewed_package in venv.package_metadata.values(): + app_paths += viewed_package.app_paths + for dep_paths in viewed_package.app_paths_of_dependencies.values(): + app_paths += dep_paths else: - # Doesn't have a valid python interpreter. We'll take our best guess on what to uninstall - # here based on symlink location. pipx doesn't use symlinks on windows, so this is for - # non-windows only. - # The heuristic here is any symlink in ~/.local/bin pointing to .local/pipx/venvs/PACKAGE/bin - # should be uninstalled. - if WINDOWS: - app_paths = [] + # fallback if not metadata from pipx_metadata.json + if venv.python_path.is_file(): + # has a valid python interpreter and can get metadata about the package + metadata = venv.get_venv_metadata_for_package(package) + app_paths = metadata.app_paths + for dep_paths in metadata.app_paths_of_dependencies.values(): + app_paths += dep_paths else: - apps_linking_to_venv_bin_dir = [ - f - for f in constants.LOCAL_BIN_DIR.iterdir() - if str(f.resolve()).startswith(str(venv.bin_path)) - ] - app_paths = apps_linking_to_venv_bin_dir + # Doesn't have a valid python interpreter. We'll take our best guess on what to uninstall + # here based on symlink location. pipx doesn't use symlinks on windows, so this is for + # non-windows only. + # The heuristic here is any symlink in ~/.local/bin pointing to .local/pipx/venvs/PACKAGE/bin + # should be uninstalled. + if WINDOWS: + app_paths = [] + else: + apps_linking_to_venv_bin_dir = [ + f + for f in constants.LOCAL_BIN_DIR.iterdir() + if str(f.resolve()).startswith(str(venv.bin_path)) + ] + app_paths = apps_linking_to_venv_bin_dir for file in local_bin_dir.iterdir(): if WINDOWS: @@ -417,10 +466,7 @@ def reinstall_all( venv_container: VenvContainer, local_bin_dir: Path, python: str, - pip_args: List[str], - venv_args: List[str], verbose: bool, - include_dependencies: bool, *, skip: List[str], ): @@ -428,22 +474,50 @@ def reinstall_all( package = venv_dir.name if package in skip: continue + + venv = Venv(venv_dir, verbose=verbose) + + if venv.pipx_metadata.main_package.package_or_url is not None: + package_or_url = venv.pipx_metadata.main_package.package_or_url + else: + package_or_url = package + uninstall(venv_dir, package, local_bin_dir, verbose) - package_or_url = package + # install main package first install( venv_dir, package, package_or_url, local_bin_dir, python, - pip_args, - venv_args, + venv.pipx_metadata.main_package.pip_args, + venv.pipx_metadata.venv_args, verbose, force=True, - include_dependencies=include_dependencies, + include_dependencies=venv.pipx_metadata.main_package.include_dependencies, ) + # now install injected packages + for ( + injected_name, + injected_package, + ) in venv.pipx_metadata.injected_packages.items(): + if injected_package.package_or_url is None: + # This should never happen, but package_or_url is type + # Optional[str] so mypy thinks it could be None + raise PipxError("Internal Error injecting package") + inject( + venv_dir, + injected_name, + injected_package.package_or_url, + injected_package.pip_args, + verbose=verbose, + include_apps=injected_package.include_apps, + include_dependencies=injected_package.include_dependencies, + force=True, + ) + def _expose_apps_globally( local_bin_dir: Path, app_paths: List[Path], package: str, *, force: bool @@ -524,22 +598,31 @@ def _get_package_summary( if not python_path.is_file(): return f" package {red(bold(package))} has invalid interpreter {str(python_path)}" - metadata = venv.get_venv_metadata_for_package(package) + package_metadata = venv.package_metadata[package] - if metadata.package_version is None: + if package_metadata.package_version is None: not_installed = red("is not installed") return f" package {bold(package)} {not_installed} in the venv {str(path)}" - apps = metadata.apps + metadata.apps_of_dependencies + apps = package_metadata.apps + package_metadata.apps_of_dependencies exposed_app_paths = _get_exposed_app_paths_for_package( venv.bin_path, apps, constants.LOCAL_BIN_DIR ) exposed_binary_names = sorted(p.name for p in exposed_app_paths) - unavailable_binary_names = sorted(set(metadata.apps) - set(exposed_binary_names)) + unavailable_binary_names = sorted( + set(package_metadata.apps) - set(exposed_binary_names) + ) + # The following is to satisfy mypy that python_version is str and not + # Optional[str] + python_version = ( + venv.pipx_metadata.python_version + if venv.pipx_metadata.python_version is not None + else "" + ) return _get_list_output( - metadata.python_version, + python_version, python_path, - metadata.package_version, + package_metadata.package_version, package, new_install, exposed_binary_names, diff --git a/src/pipx/commands/upgrade.py b/src/pipx/commands/upgrade.py index 0ea15f65ef..00804fca46 100644 --- a/src/pipx/commands/upgrade.py +++ b/src/pipx/commands/upgrade.py @@ -5,7 +5,6 @@ from pipx import constants -from pipx.constants import PIPX_PACKAGE_NAME from pipx.emojies import sleep from pipx.util import PipxError @@ -16,12 +15,10 @@ def upgrade( venv_dir: Path, package: str, - package_or_url: str, pip_args: List[str], verbose: bool, *, upgrading_all: bool, - include_dependencies: bool, force: bool, ) -> int: """Returns nonzero if package was upgraded, 0 if version did not change""" @@ -34,21 +31,44 @@ def upgrade( venv = Venv(venv_dir, verbose=verbose) - old_version = venv.get_venv_metadata_for_package(package).package_version + package_metadata = venv.package_metadata[package] + if package_metadata.package_or_url is not None: + package_or_url = package_metadata.package_or_url + old_version = package_metadata.package_version + include_apps = package_metadata.include_apps + include_dependencies = package_metadata.include_dependencies + else: + # fallback if no metadata + package_or_url = package + old_version = "" + include_apps = True + include_dependencies = False + + if package == "pipx": + package_or_url = "pipx" # Upgrade shared libraries (pip, setuptools and wheel) venv.upgrade_packaging_libraries(pip_args) - venv.upgrade_package(package_or_url, pip_args) - new_version = venv.get_venv_metadata_for_package(package).package_version + venv.upgrade_package( + package, + package_or_url, + pip_args, + include_dependencies=include_dependencies, + include_apps=include_apps, + is_main_package=True, + ) + # TODO 20191026: upgrade injected packages also (Issue #79) + + package_metadata = venv.package_metadata[package] + new_version = package_metadata.package_version - metadata = venv.get_venv_metadata_for_package(package) _expose_apps_globally( - constants.LOCAL_BIN_DIR, metadata.app_paths, package, force=force + constants.LOCAL_BIN_DIR, package_metadata.app_paths, package, force=force ) if include_dependencies: - for _, app_paths in metadata.app_paths_of_dependencies.items(): + for _, app_paths in package_metadata.app_paths_of_dependencies.items(): _expose_apps_globally( constants.LOCAL_BIN_DIR, app_paths, package, force=force ) @@ -69,36 +89,26 @@ def upgrade( def upgrade_all( - venv_container: VenvContainer, - pip_args: List[str], - verbose: bool, - *, - include_dependencies: bool, - skip: List[str], - force: bool, + venv_container: VenvContainer, verbose: bool, *, skip: List[str], force: bool ): packages_upgraded = 0 num_packages = 0 for venv_dir in venv_container.iter_venv_dirs(): num_packages += 1 package = venv_dir.name + venv = Venv(venv_dir, verbose=verbose) if package in skip: continue - if package == "pipx": - package_or_url = PIPX_PACKAGE_NAME - else: - package_or_url = package try: packages_upgraded += upgrade( venv_dir, package, - package_or_url, - pip_args, + venv.pipx_metadata.main_package.pip_args, verbose, upgrading_all=True, - include_dependencies=include_dependencies, force=force, ) + except Exception: logging.error(f"Error encountered when upgrading {package}") diff --git a/src/pipx/constants.py b/src/pipx/constants.py index f2cd5acec7..861090da85 100644 --- a/src/pipx/constants.py +++ b/src/pipx/constants.py @@ -15,7 +15,6 @@ PIPX_SHARED_PTH = "pipx_shared.pth" LOCAL_BIN_DIR = Path(os.environ.get("PIPX_BIN_DIR", DEFAULT_PIPX_BIN_DIR)).resolve() PIPX_VENV_CACHEDIR = PIPX_HOME / ".cache" -PIPX_PACKAGE_NAME = "pipx" TEMP_VENV_EXPIRATION_THRESHOLD_DAYS = 14 try: WindowsError diff --git a/src/pipx/main.py b/src/pipx/main.py index 608b02c844..df0ce32115 100644 --- a/src/pipx/main.py +++ b/src/pipx/main.py @@ -178,6 +178,7 @@ def run_pipx_command(args): # noqa: C901 commands.inject( venv_dir, dep, + dep, # NOP now, but in future can be package_or_url from --spec pip_args, verbose=verbose, include_apps=args.include_apps, @@ -185,18 +186,8 @@ def run_pipx_command(args): # noqa: C901 force=args.force, ) elif args.command == "upgrade": - package_or_url = ( - args.spec if ("spec" in args and args.spec is not None) else package - ) return commands.upgrade( - venv_dir, - package, - package_or_url, - pip_args, - verbose, - upgrading_all=False, - include_dependencies=args.include_deps, - force=args.force, + venv_dir, package, pip_args, verbose, upgrading_all=False, force=args.force ) elif args.command == "list": return commands.list_packages(venv_container) @@ -206,22 +197,14 @@ def run_pipx_command(args): # noqa: C901 return commands.uninstall_all(venv_container, constants.LOCAL_BIN_DIR, verbose) elif args.command == "upgrade-all": return commands.upgrade_all( - venv_container, - pip_args, - verbose, - include_dependencies=args.include_deps, - skip=args.skip, - force=args.force, + venv_container, verbose, skip=args.skip, force=args.force ) elif args.command == "reinstall-all": return commands.reinstall_all( venv_container, constants.LOCAL_BIN_DIR, args.python, - pip_args, - venv_args, verbose, - args.include_deps, skip=args.skip, ) elif args.command == "runpip": @@ -337,14 +320,12 @@ def _add_upgrade(subparsers, autocomplete_list_of_installed_packages): description="Upgrade a package in a pipx-managed Virtual Environment by running 'pip install --upgrade PACKAGE'", ) p.add_argument("package").completer = autocomplete_list_of_installed_packages - p.add_argument("--spec", help=SPEC_HELP) p.add_argument( "--force", "-f", action="store_true", help="Modify existing virtual environment and files in PIPX_BIN_DIR", ) - add_include_dependencies(p) add_pip_venv_args(p) p.add_argument("--verbose", action="store_true") @@ -357,8 +338,6 @@ def _add_upgrade_all(subparsers): description="Upgrades all packages within their virtual environments by running 'pip install --upgrade PACKAGE'", ) - add_include_dependencies(p) - add_pip_venv_args(p) p.add_argument("--skip", nargs="+", default=[], help="skip these packages") p.add_argument( "--force", @@ -397,13 +376,11 @@ def _add_reinstall_all(subparsers): """ Reinstalls all packages. - Packages are uninstalled, then installed with pipx install PACKAGE. + Packages are uninstalled, then installed with pipx install PACKAGE + with the same options used in the original install of PACKAGE. This is useful if you upgraded to a new version of Python and want all your packages to use the latest as well. - If you originally installed a package from a source other than PyPI, - this command may behave in unexpected ways since it will reinstall from PyPI. - """ ), ) @@ -415,8 +392,6 @@ def _add_reinstall_all(subparsers): "and run the associated app/apps. Must be v3.5+." ), ) - add_include_dependencies(p) - add_pip_venv_args(p) p.add_argument("--skip", nargs="+", default=[], help="skip these packages") p.add_argument("--verbose", action="store_true") diff --git a/src/pipx/pipx_metadata_file.py b/src/pipx/pipx_metadata_file.py new file mode 100644 index 0000000000..64526c3c47 --- /dev/null +++ b/src/pipx/pipx_metadata_file.py @@ -0,0 +1,157 @@ +import json +import logging +from pathlib import Path +import textwrap +from typing import List, Dict, NamedTuple, Any, Optional + +from pipx.util import PipxError + + +PIPX_INFO_FILENAME = "pipx_metadata.json" + + +class JsonEncoderHandlesPath(json.JSONEncoder): + def default(self, obj): + # only handles what json.JSONEncoder doesn't understand by default + if isinstance(obj, Path): + return {"__type__": "Path", "__Path__": str(obj)} + return super().default(obj) + + +def _json_decoder_object_hook(json_dict): + if json_dict.get("__type__", None) == "Path" and "__Path__" in json_dict: + return Path(json_dict["__Path__"]) + return json_dict + + +class PackageInfo(NamedTuple): + package: Optional[str] + package_or_url: Optional[str] + pip_args: List[str] + include_dependencies: bool + include_apps: bool + apps: List[str] + app_paths: List[Path] + apps_of_dependencies: List[str] + app_paths_of_dependencies: Dict[str, List[Path]] + package_version: str + + +class PipxMetadata: + def __init__(self, venv_dir: Path, read: bool = True): + self.venv_dir = venv_dir + # We init this instance with reasonable fallback defaults for all + # members, EXCEPT for those we cannot know: + # self.main_package.package=None + # self.main_package.package_or_url=None + # self.python_version=None + self.main_package = PackageInfo( + package=None, + package_or_url=None, + pip_args=[], + include_dependencies=False, + include_apps=True, # always True for main_package + apps=[], + app_paths=[], + apps_of_dependencies=[], + app_paths_of_dependencies={}, + package_version="", + ) + self.python_version: Optional[str] = None + self.venv_args: List[str] = [] + self.injected_packages: Dict[str, PackageInfo] = {} + + # Only change this if file format changes + self._pipx_metadata_version: str = "0.1" + + if read: + self.read() + + def reset(self) -> None: + # We init this instance with reasonable fallback defaults for all + # members, EXCEPT for those we cannot know: + # self.main_package.package_or_url=None + # self.venv_metadata.package_or_url=None + self.main_package = PackageInfo( + package=None, + package_or_url=None, + pip_args=[], + include_dependencies=False, + include_apps=True, # always True for main_package + apps=[], + app_paths=[], + apps_of_dependencies=[], + app_paths_of_dependencies={}, + package_version="", + ) + self.python_version = None + self.venv_args = [] + self.injected_packages = {} + + def to_dict(self) -> Dict[str, Any]: + return { + "main_package": self.main_package._asdict(), + "python_version": self.python_version, + "venv_args": self.venv_args, + "injected_packages": { + name: data._asdict() for (name, data) in self.injected_packages.items() + }, + "pipx_metadata_version": self._pipx_metadata_version, + } + + def from_dict(self, input_dict: Dict[str, Any]) -> None: + self.main_package = PackageInfo(**input_dict["main_package"]) + self.python_version = input_dict["python_version"] + self.venv_args = input_dict["venv_args"] + self.injected_packages = { + name: PackageInfo(**data) + for (name, data) in input_dict["injected_packages"].items() + } + + def _validate_before_write(self): + if ( + self.main_package.package is None + or self.main_package.package_or_url is None + or not self.main_package.include_apps + ): + raise PipxError("Internal Error: PipxMetadata is corrupt, cannot write.") + + def write(self) -> None: + self._validate_before_write() + try: + with open(self.venv_dir / PIPX_INFO_FILENAME, "w") as pipx_metadata_fh: + json.dump( + self.to_dict(), + pipx_metadata_fh, + indent=4, + sort_keys=True, + cls=JsonEncoderHandlesPath, + ) + except IOError: + logging.warning( + textwrap.fill( + f"Unable to write {PIPX_INFO_FILENAME} to {self.venv_dir}. " + f"This may cause future pipx operations involving " + f"{self.venv_dir.name} to fail or behave incorrectly.", + width=79, + ) + ) + pass + + def read(self, verbose: bool = False) -> None: + try: + with open(self.venv_dir / PIPX_INFO_FILENAME, "r") as pipx_metadata_fh: + self.from_dict( + json.load(pipx_metadata_fh, object_hook=_json_decoder_object_hook) + ) + except IOError: # Reset self if problem reading + if verbose: + logging.warning( + textwrap.fill( + f"Unable to read {PIPX_INFO_FILENAME} in {self.venv_dir}. " + f"This may cause this or future pipx operations involving " + f"{self.venv_dir.name} to fail or behave incorrectly.", + width=79, + ) + ) + return diff --git a/src/pipx/util.py b/src/pipx/util.py index 21a25e2cc6..8cd7a95aee 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -91,15 +91,39 @@ def get_site_packages(python: Path) -> Path: return Path(output.strip()) -def run(cmd: Sequence[Union[str, Path]], check=True) -> int: - """Run arbitrary command as subprocess""" - +def run_subprocess( + cmd: Sequence[Union[str, Path]], + capture_stdout: bool = True, + capture_stderr: bool = True, +) -> subprocess.CompletedProcess: + """Run arbitrary command as subprocess, capturing stderr and stout""" + + # Null out PYTHONPATH because some platforms (macOS with Homebrew) add + # pipx directories to it, and can make it appear to venvs as though + # pipx dependencies are in the venv path (#233) env = {k: v for k, v in os.environ.items() if k.upper() != "PYTHONPATH"} cmd_str = " ".join(str(c) for c in cmd) logging.info(f"running {cmd_str}") # windows cannot take Path objects, only strings cmd_str_list = [str(c) for c in cmd] - returncode = subprocess.run(cmd_str_list, env=env).returncode + return subprocess.run( + cmd_str_list, + env=env, + stdout=subprocess.PIPE if capture_stdout else None, + stderr=subprocess.PIPE if capture_stderr else None, + universal_newlines=True, # implies decoded strings in stdout, stderr + ) + + +def run(cmd: Sequence[Union[str, Path]], check=True) -> int: + """Run arbitrary command as subprocess""" + + returncode = run_subprocess( + cmd, capture_stdout=False, capture_stderr=False + ).returncode + + cmd_str = " ".join(str(c) for c in cmd) + if check and returncode: raise PipxError(f"{cmd_str!r} failed") return returncode diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 501a0816a2..4548db4beb 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -1,12 +1,13 @@ import json import logging import pkgutil -import subprocess +import re from pathlib import Path -from typing import Dict, Generator, List, NamedTuple +from typing import Generator, List, NamedTuple, Dict, Set, Optional from pipx.animate import animate from pipx.constants import DEFAULT_PYTHON, PIPX_SHARED_PTH, WINDOWS +from pipx.pipx_metadata_file import PipxMetadata, PackageInfo from pipx.shared_libs import shared_libs from pipx.util import ( PipxError, @@ -15,9 +16,14 @@ get_venv_paths, rmdir, run, + run_subprocess, ) +class PackageInstallFailureError(Exception): + pass + + class VenvContainer: """A collection of venvs managed by pipx. """ @@ -49,7 +55,7 @@ def verify_shared_libs(self): Venv(p) -class PipxVenvMetadata(NamedTuple): +class VenvMetadata(NamedTuple): apps: List[str] app_paths: List[Path] apps_of_dependencies: List[str] @@ -75,6 +81,7 @@ def __init__( self.root = path self._python = python self.bin_path, self.python_path = get_venv_paths(self.root) + self.pipx_metadata = PipxMetadata(venv_dir=path) self.verbose = verbose self.do_animation = not verbose try: @@ -99,7 +106,7 @@ def __init__( ) @property - def uses_shared_libs(self): + def uses_shared_libs(self) -> bool: if self._existing: pth_files = self.root.glob("**/" + PIPX_SHARED_PTH) return next(pth_files, None) is not None @@ -107,6 +114,15 @@ def uses_shared_libs(self): # always use shared libs when creating a new venv return True + @property + def package_metadata(self) -> Dict[str, PackageInfo]: + return_dict = self.pipx_metadata.injected_packages.copy() + if self.pipx_metadata.main_package.package is not None: + return_dict[ + self.pipx_metadata.main_package.package + ] = self.pipx_metadata.main_package + return return_dict + def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: with animate("creating virtual environment", self.do_animation): cmd = [self._python, "-m", "venv", "--without-pip"] @@ -124,6 +140,9 @@ def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: # its contents are additional items (one per line) to be added to sys.path pipx_pth.write_text(str(shared_libs.site_packages) + "\n", encoding="utf-8") + self.pipx_metadata.venv_args = venv_args + self.pipx_metadata.python_version = self.get_python_version() + def safe_to_remove(self) -> bool: return not self._existing @@ -142,16 +161,58 @@ def upgrade_packaging_libraries(self, pip_args: List[str]) -> None: else: # TODO: setuptools and wheel? Original code didn't bother # but shared libs code does. - self.upgrade_package("pip", pip_args) + self._upgrade_package_no_metadata("pip", pip_args) + + def install_package( + self, + package: Optional[str], # if None, will be determined in this function + package_or_url: str, + pip_args: List[str], + include_dependencies: bool, + include_apps: bool, + is_main_package: bool, + ) -> None: - def install_package(self, package_or_url: str, pip_args: List[str]) -> None: with animate(f"installing package {package_or_url!r}", self.do_animation): if pip_args is None: pip_args = [] + if package is None: + # If no package name is supplied, install only main package + # first in order to see what its name is + old_package_set = self.list_installed_packages() + cmd = ["install"] + pip_args + ["--no-dependencies"] + [package_or_url] + self._run_pip(cmd) + installed_packages = self.list_installed_packages() - old_package_set + if len(installed_packages) == 1: + package = installed_packages.pop() + logging.info(f"Determined package name: '{package}'") + else: + package = None cmd = ["install"] + pip_args + [package_or_url] self._run_pip(cmd) - def get_venv_metadata_for_package(self, package: str) -> PipxVenvMetadata: + if package is None: + logging.warning( + f"Cannot determine package name for package_or_url='{package_or_url}'. " + f"Unable to retrieve package metadata. " + f"Unable to verify if package was installed properly." + ) + return + + self._update_package_metadata( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=include_apps, + is_main_package=is_main_package, + ) + + # Verify package installed ok + if self.package_metadata[package].package_version is None: + raise PackageInstallFailureError + + def get_venv_metadata_for_package(self, package: str) -> VenvMetadata: data = json.loads( get_script_output( self.python_path, VENV_METADATA_INSPECTOR, package, str(self.bin_path) @@ -175,28 +236,149 @@ def get_venv_metadata_for_package(self, package: str) -> PipxVenvMetadata: data["app_paths_of_dependencies"][dep] = paths data["apps_of_dependencies"] += [path.name for path in paths] - return PipxVenvMetadata(**data) + return VenvMetadata(**data) + + def _update_package_metadata( + self, + package: str, + package_or_url: str, + pip_args: List[str], + include_dependencies: bool, + include_apps: bool, + is_main_package: bool, + ) -> None: + venv_package_metadata = self.get_venv_metadata_for_package(package) + package_info = PackageInfo( + package=package, + package_or_url=abs_path_if_local(package_or_url, self, pip_args), + pip_args=pip_args, + include_apps=include_apps, + include_dependencies=include_dependencies, + apps=venv_package_metadata.apps, + app_paths=venv_package_metadata.app_paths, + apps_of_dependencies=venv_package_metadata.apps_of_dependencies, + app_paths_of_dependencies=venv_package_metadata.app_paths_of_dependencies, + package_version=venv_package_metadata.package_version, + ) + if is_main_package: + self.pipx_metadata.main_package = package_info + else: + self.pipx_metadata.injected_packages[package] = package_info + + self.pipx_metadata.write() def get_python_version(self) -> str: - return ( - subprocess.run([str(self.python_path), "--version"], stdout=subprocess.PIPE) - .stdout.decode() - .strip() + return run_subprocess([str(self.python_path), "--version"]).stdout.strip() + + def pip_search(self, search_term: str, pip_search_args: List[str]) -> str: + cmd_run = run_subprocess( + [str(self.python_path), "-m", "pip", "search"] + + pip_search_args + + [search_term] ) + return cmd_run.stdout.strip() - def run_app(self, app: str, app_args: List[str]): + def list_installed_packages(self) -> Set[str]: + cmd_run = run_subprocess( + [str(self.python_path), "-m", "pip", "list", "--format=json"] + ) + pip_list = json.loads(cmd_run.stdout.strip()) + return set([x["name"] for x in pip_list]) + + def run_app(self, app: str, app_args: List[str]) -> int: cmd = [str(self.bin_path / app)] + app_args try: return run(cmd, check=False) except KeyboardInterrupt: - pass + return 130 # shell code for Ctrl-C + + def _upgrade_package_no_metadata( + self, package_or_url: str, pip_args: List[str] + ) -> None: + with animate(f"upgrading package {package_or_url!r}", self.do_animation): + self._run_pip(["install"] + pip_args + ["--upgrade", package_or_url]) - def upgrade_package(self, package_or_url: str, pip_args: List[str]): + def upgrade_package( + self, + package: str, + package_or_url: str, + pip_args: List[str], + include_dependencies: bool, + include_apps: bool, + is_main_package: bool, + ) -> None: with animate(f"upgrading package {package_or_url!r}", self.do_animation): self._run_pip(["install"] + pip_args + ["--upgrade", package_or_url]) - def _run_pip(self, cmd): - cmd = [self.python_path, "-m", "pip"] + cmd + self._update_package_metadata( + package=package, + package_or_url=package_or_url, + pip_args=pip_args, + include_dependencies=include_dependencies, + include_apps=include_apps, + is_main_package=is_main_package, + ) + + def _run_pip(self, cmd: List[str]) -> int: + cmd = [str(self.python_path), "-m", "pip"] + cmd if not self.verbose: cmd.append("-q") return run(cmd) + + +def abs_path_if_local(package_or_url: str, venv: Venv, pip_args: List[str]) -> str: + """Return the absolute path if package_or_url represents a filepath + and not a pypi package + """ + pkg_path = Path(package_or_url) + if not pkg_path.exists(): + # no existing path, must be pypi package or non-existent + return package_or_url + + # Editable packages are either local or url, non-url must be local. + # https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs + if "--editable" in pip_args and pkg_path.exists(): + return str(pkg_path.resolve()) + + # https://www.python.org/dev/peps/pep-0508/#names + valid_pkg_name = bool( + re.search(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_or_url, re.I) + ) + if not valid_pkg_name: + return str(pkg_path.resolve()) + + # If all of the above conditions do not return, we may have used a pypi + # package. + # If we find a pypi package with this name installed, assume we just + # installed it. + pip_search_args: List[str] + + # If user-defined pypi index url, then use it for search + try: + arg_i = pip_args.index("--index-url") + except ValueError: + pip_search_args = [] + else: + pip_search_args = pip_args[arg_i : arg_i + 2] + + pip_search_result_str = venv.pip_search(package_or_url, pip_search_args) + pip_search_results = pip_search_result_str.split("\n") + + # Get package_or_url and following related lines from pip search stdout + pkg_found = False + pip_search_found = [] + for pip_search_line in pip_search_results: + if pkg_found: + if re.search(r"^\s", pip_search_line): + pip_search_found.append(pip_search_line) + else: + break + elif pip_search_line.startswith(package_or_url): + pip_search_found.append(pip_search_line) + pkg_found = True + pip_found_str = " ".join(pip_search_found) + + if pip_found_str.startswith(package_or_url) and "INSTALLED" in pip_found_str: + return package_or_url + else: + return str(pkg_path.resolve()) diff --git a/tests/test_animate.py b/tests/test_animate.py index 0389fafeb9..acba0966bb 100644 --- a/tests/test_animate.py +++ b/tests/test_animate.py @@ -19,7 +19,7 @@ def check_animate_output( chars_to_test = 1 + len("".join(frame_strings[:frames_to_test])) with pipx.animate.animate(test_string, do_animation=True): - time.sleep(frame_period * (frames_to_test - 1) + 0.2) + time.sleep(frame_period * (frames_to_test - 1) + 0.5) captured = capsys.readouterr() assert captured.err[:chars_to_test] == expected_string[:chars_to_test] diff --git a/tests/test_pipx_metadata_file.py b/tests/test_pipx_metadata_file.py new file mode 100644 index 0000000000..ee1591eb62 --- /dev/null +++ b/tests/test_pipx_metadata_file.py @@ -0,0 +1,178 @@ +from pathlib import Path +import pytest # type: ignore + +from helpers import assert_not_in_virtualenv, run_pipx_cli +import pipx.constants +from pipx.pipx_metadata_file import PipxMetadata, PackageInfo +from pipx.util import PipxError + +assert_not_in_virtualenv() + + +TEST_PACKAGE1 = PackageInfo( + package="test_package", + package_or_url="test_package_url", + pip_args=[], + include_apps=True, + include_dependencies=False, + apps=["testapp"], + app_paths=[Path("/usr/bin")], + apps_of_dependencies=["dep1"], + app_paths_of_dependencies={"dep1": [Path("bin")]}, + package_version="0.1.2", +) +TEST_PACKAGE2 = PackageInfo( + package="inj_package", + package_or_url="inj_package_url", + pip_args=["-e"], + include_apps=True, + include_dependencies=False, + apps=["injapp"], + app_paths=[Path("/usr/bin")], + apps_of_dependencies=["dep2"], + app_paths_of_dependencies={"dep2": [Path("bin")]}, + package_version="6.7.8", +) + +# Reference metadata for various packages +PYCOWSAY_PACKAGE_REF = PackageInfo( + package="pycowsay", + package_or_url="pycowsay", + pip_args=[], + include_dependencies=False, + include_apps=True, + apps=["pycowsay"], + app_paths=[Path("pycowsay/bin/pycowsay")], # Placeholder, not real path + apps_of_dependencies=[], + app_paths_of_dependencies={}, + package_version="0.0.0.1", +) +BLACK_PACKAGE_REF = PackageInfo( + package="black", + package_or_url="black", + pip_args=[], + include_dependencies=False, + include_apps=True, + apps=["pycowsay"], + app_paths=[Path("black/bin/black")], # Placeholder, not real path + apps_of_dependencies=[], + app_paths_of_dependencies={}, + package_version="19.10b3", +) + + +def test_pipx_metadata_file_create(tmp_path): + pipx_metadata = PipxMetadata(tmp_path) + pipx_metadata.main_package = TEST_PACKAGE1 + pipx_metadata.python_version = "3.4.5" + pipx_metadata.venv_args = ["--system-site-packages"] + pipx_metadata.injected_packages = {"injected": TEST_PACKAGE2} + pipx_metadata.write() + + pipx_metadata2 = PipxMetadata(tmp_path) + + for attribute in [ + "venv_dir", + "main_package", + "python_version", + "venv_args", + "injected_packages", + ]: + assert getattr(pipx_metadata, attribute) == getattr(pipx_metadata2, attribute) + + +@pytest.mark.parametrize( + "test_package", + [ + TEST_PACKAGE1._replace(include_apps=False), + TEST_PACKAGE1._replace(package=None), + TEST_PACKAGE1._replace(package_or_url=None), + ], +) +def test_pipx_metadata_file_validation(tmp_path, test_package): + venv_dir = tmp_path / "venv" + venv_dir.mkdir() + + pipx_metadata = PipxMetadata(venv_dir) + pipx_metadata.main_package = test_package + pipx_metadata.python_version = "3.4.5" + pipx_metadata.venv_args = ["--system-site-packages"] + pipx_metadata.injected_packages = {} + + with pytest.raises(PipxError): + pipx_metadata.write() + + +def assert_package_metadata(test_metadata, ref_metadata): + # update package version of ref with recent package version + # only compare sets for apps, app_paths so order is not important + + assert test_metadata.package_version != "" + assert isinstance(test_metadata.apps, list) + assert isinstance(test_metadata.app_paths, list) + + test_metadata_replaced = test_metadata._replace( + apps=set(test_metadata.apps), app_paths=set(test_metadata.apps) + ) + ref_metadata_replaced = ref_metadata._replace( + apps=set(ref_metadata.apps), + app_paths=set(ref_metadata.apps), + package_version=test_metadata.package_version, + ) + assert test_metadata_replaced == ref_metadata_replaced + + +def test_package_install(monkeypatch, tmp_path, pipx_temp_env): + pipx_venvs_dir = pipx.constants.PIPX_HOME / "venvs" + + run_pipx_cli(["install", "pycowsay"]) + assert (pipx_venvs_dir / "pycowsay" / "pipx_metadata.json").is_file() + + pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") + + if pipx.constants.WINDOWS: + ref_replacement_fields = { + "app_paths": [pipx_venvs_dir / "pycowsay" / "Scripts" / "pycowsay.exe"], + "apps": ["pycowsay", "pycowsay.exe"], + } + else: + ref_replacement_fields = { + "app_paths": [pipx_venvs_dir / "pycowsay" / "bin" / "pycowsay"] + } + assert_package_metadata( + pipx_metadata.main_package, + PYCOWSAY_PACKAGE_REF._replace(include_apps=True, **ref_replacement_fields), + ) + + +def test_package_inject(monkeypatch, tmp_path, pipx_temp_env): + pipx_venvs_dir = pipx.constants.PIPX_HOME / "venvs" + + run_pipx_cli(["install", "pycowsay"]) + run_pipx_cli(["inject", "pycowsay", "black"]) + assert (pipx_venvs_dir / "pycowsay" / "pipx_metadata.json").is_file() + + pipx_metadata = PipxMetadata(pipx_venvs_dir / "pycowsay") + + assert pipx_metadata.injected_packages.keys() == {"black"} + + if pipx.constants.WINDOWS: + ref_replacement_fields = { + "apps": ["black", "black.exe", "blackd", "blackd.exe"], + "app_paths": [ + pipx_venvs_dir / "pycowsay" / "Scripts" / "black.exe", + pipx_venvs_dir / "pycowsay" / "Scripts" / "blackd.exe", + ], + } + else: + ref_replacement_fields = { + "apps": ["black", "blackd"], + "app_paths": [ + pipx_venvs_dir / "pycowsay" / "bin" / "black", + pipx_venvs_dir / "pycowsay" / "bin" / "blackd", + ], + } + assert_package_metadata( + pipx_metadata.injected_packages["black"], + BLACK_PACKAGE_REF._replace(include_apps=False, **ref_replacement_fields), + ) diff --git a/tests/test_run.py b/tests/test_run.py index 284d3d0be8..6bf355fe05 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -68,3 +68,29 @@ def test_run_ensure_null_pythonpath(): stderr=subprocess.PIPE, ).stdout ) + + +# packages listed roughly in order of increasing test duration +@pytest.mark.parametrize( + "package, package_or_url, app_args", + [ + ("pycowsay", "pycowsay", ["pycowsay", "hello"]), + ("shell-functools", "shell-functools", ["filter", "--help"]), + ("black", "black", ["black", "--help"]), + ("pylint", "pylint", ["pylint", "--help"]), + ("kaggle", "kaggle", ["kaggle", "--help"]), + ("ipython", "ipython", ["ipython", "--version"]), + ("cloudtoken", "cloudtoken", ["cloudtoken", "--help"]), + ("awscli", "awscli", ["aws", "--help"]), + # ("ansible", "ansible", ["ansible", "--help"]), # takes too long + ], +) +def test_package_determination( + caplog, pipx_temp_env, package, package_or_url, app_args +): + caplog.set_level(logging.INFO) + + run_pipx_cli(["run", "--verbose", "--spec", package_or_url, "--"] + app_args) + + assert "Cannot determine package name" not in caplog.text + assert f"Determined package name: '{package}'" in caplog.text