From 0cd009b5a1338f66397f71c85a75f576a2f3eabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sun, 21 Jun 2020 08:28:50 +0100 Subject: [PATCH] Implement periodic update feature (#1841) Co-authored-by: Pradyun Gedam --- .coveragerc | 11 +- .pre-commit-config.yaml | 14 +- codecov.yaml | 5 + docs/changelog/1821.doc.rst | 1 + docs/changelog/1821.feature.rst | 8 + docs/changelog/1841.feature.rst | 7 + docs/cli_interface.rst | 2 +- docs/user_guide.rst | 64 +++- setup.cfg | 6 +- setup.py | 5 +- src/virtualenv/__main__.py | 41 +- .../{run/app_data.py => app_data/__init__.py} | 47 +-- src/virtualenv/app_data/base.py | 91 +++++ src/virtualenv/app_data/na.py | 67 ++++ src/virtualenv/app_data/via_disk_folder.py | 172 +++++++++ src/virtualenv/app_data/via_tempdir.py | 28 ++ src/virtualenv/config/cli/parser.py | 1 - src/virtualenv/create/creator.py | 9 +- src/virtualenv/{ => create}/pyenv_cfg.py | 0 src/virtualenv/create/via_global_ref/api.py | 3 +- src/virtualenv/create/via_global_ref/store.py | 7 +- src/virtualenv/create/via_global_ref/venv.py | 2 +- src/virtualenv/discovery/builtin.py | 2 +- src/virtualenv/discovery/cached_py_info.py | 82 ++-- src/virtualenv/report.py | 1 + src/virtualenv/run/__init__.py | 70 +++- src/virtualenv/run/plugin/discovery.py | 6 - src/virtualenv/{ => run}/session.py | 0 src/virtualenv/seed/embed/__init__.py | 1 - src/virtualenv/seed/embed/base_embed.py | 74 ++-- src/virtualenv/seed/embed/pip_invoke.py | 48 +-- .../seed/{ => embed}/via_app_data/__init__.py | 0 .../via_app_data/pip_install/__init__.py | 0 .../via_app_data/pip_install/base.py | 4 +- .../via_app_data/pip_install/copy.py | 0 .../via_app_data/pip_install/symlink.py | 0 .../seed/embed/via_app_data/via_app_data.py | 127 +++++++ src/virtualenv/seed/embed/wheels/acquire.py | 178 --------- .../seed/via_app_data/via_app_data.py | 128 ------- src/virtualenv/seed/wheels/__init__.py | 11 + src/virtualenv/seed/wheels/acquire.py | 114 ++++++ src/virtualenv/seed/wheels/bundle.py | 51 +++ .../wheels => wheels/embed}/__init__.py | 24 +- .../embed}/pip-19.1.1-py2.py3-none-any.whl | Bin .../embed}/pip-20.1.1-py2.py3-none-any.whl | Bin .../setuptools-43.0.0-py2.py3-none-any.whl | Bin .../setuptools-44.1.1-py2.py3-none-any.whl | Bin .../embed}/setuptools-47.1.1-py3-none-any.whl | Bin .../embed}/wheel-0.33.6-py2.py3-none-any.whl | Bin .../embed}/wheel-0.34.2-py2.py3-none-any.whl | Bin src/virtualenv/seed/wheels/periodic_update.py | 311 ++++++++++++++++ src/virtualenv/seed/wheels/util.py | 116 ++++++ src/virtualenv/{ => util}/error.py | 0 src/virtualenv/util/lock.py | 14 +- .../util/path/_pathlib/via_os_path.py | 8 +- src/virtualenv/util/subprocess/__init__.py | 4 + src/virtualenv/util/zipapp.py | 25 +- tasks/upgrade_wheels.py | 54 ++- tests/conftest.py | 23 +- tests/integration/test_zipapp.py | 3 +- tests/unit/activation/conftest.py | 5 +- tests/unit/activation/test_activate_this.py | 2 +- tests/unit/activation/test_xonsh.py | 2 + tests/unit/config/test___main__.py | 70 ++++ tests/unit/create/conftest.py | 4 +- tests/unit/create/test_creator.py | 6 +- .../via_global_ref}/greet/greet2.c | 0 .../via_global_ref}/greet/greet3.c | 0 .../via_global_ref}/greet/setup.py | 0 .../via_global_ref/test_build_c_ext.py} | 0 .../unit/seed/{ => embed}/test_base_embed.py | 0 .../test_boostrap_link_via_app_data.py | 7 +- tests/unit/seed/embed/test_pip_invoke.py | 89 +++++ tests/unit/seed/embed/wheels/test_acquire.py | 11 - tests/unit/seed/test_pip_invoke.py | 63 ---- tests/unit/seed/wheels/test_acquire.py | 66 ++++ .../seed/wheels/test_acquire_find_wheel.py | 30 ++ .../unit/seed/wheels/test_periodic_update.py | 351 ++++++++++++++++++ tests/unit/seed/wheels/test_wheels_util.py | 31 ++ tox.ini | 6 +- 80 files changed, 2161 insertions(+), 652 deletions(-) create mode 100644 docs/changelog/1821.doc.rst create mode 100644 docs/changelog/1821.feature.rst create mode 100644 docs/changelog/1841.feature.rst rename src/virtualenv/{run/app_data.py => app_data/__init__.py} (59%) create mode 100644 src/virtualenv/app_data/base.py create mode 100644 src/virtualenv/app_data/na.py create mode 100644 src/virtualenv/app_data/via_disk_folder.py create mode 100644 src/virtualenv/app_data/via_tempdir.py rename src/virtualenv/{ => create}/pyenv_cfg.py (100%) rename src/virtualenv/{ => run}/session.py (100%) rename src/virtualenv/seed/{ => embed}/via_app_data/__init__.py (100%) rename src/virtualenv/seed/{ => embed}/via_app_data/pip_install/__init__.py (100%) rename src/virtualenv/seed/{ => embed}/via_app_data/pip_install/base.py (97%) rename src/virtualenv/seed/{ => embed}/via_app_data/pip_install/copy.py (100%) rename src/virtualenv/seed/{ => embed}/via_app_data/pip_install/symlink.py (100%) create mode 100644 src/virtualenv/seed/embed/via_app_data/via_app_data.py delete mode 100644 src/virtualenv/seed/embed/wheels/acquire.py delete mode 100644 src/virtualenv/seed/via_app_data/via_app_data.py create mode 100644 src/virtualenv/seed/wheels/__init__.py create mode 100644 src/virtualenv/seed/wheels/acquire.py create mode 100644 src/virtualenv/seed/wheels/bundle.py rename src/virtualenv/seed/{embed/wheels => wheels/embed}/__init__.py (68%) rename src/virtualenv/seed/{embed/wheels => wheels/embed}/pip-19.1.1-py2.py3-none-any.whl (100%) rename src/virtualenv/seed/{embed/wheels => wheels/embed}/pip-20.1.1-py2.py3-none-any.whl (100%) rename src/virtualenv/seed/{embed/wheels => wheels/embed}/setuptools-43.0.0-py2.py3-none-any.whl (100%) rename src/virtualenv/seed/{embed/wheels => wheels/embed}/setuptools-44.1.1-py2.py3-none-any.whl (100%) rename src/virtualenv/seed/{embed/wheels => wheels/embed}/setuptools-47.1.1-py3-none-any.whl (100%) rename src/virtualenv/seed/{embed/wheels => wheels/embed}/wheel-0.33.6-py2.py3-none-any.whl (100%) rename src/virtualenv/seed/{embed/wheels => wheels/embed}/wheel-0.34.2-py2.py3-none-any.whl (100%) create mode 100644 src/virtualenv/seed/wheels/periodic_update.py create mode 100644 src/virtualenv/seed/wheels/util.py rename src/virtualenv/{ => util}/error.py (100%) rename tests/unit/{seed => create/via_global_ref}/greet/greet2.c (100%) rename tests/unit/{seed => create/via_global_ref}/greet/greet3.c (100%) rename tests/unit/{seed => create/via_global_ref}/greet/setup.py (100%) rename tests/unit/{seed/test_extra_install.py => create/via_global_ref/test_build_c_ext.py} (100%) rename tests/unit/seed/{ => embed}/test_base_embed.py (100%) rename tests/unit/seed/{ => embed}/test_boostrap_link_via_app_data.py (95%) create mode 100644 tests/unit/seed/embed/test_pip_invoke.py delete mode 100644 tests/unit/seed/embed/wheels/test_acquire.py delete mode 100644 tests/unit/seed/test_pip_invoke.py create mode 100644 tests/unit/seed/wheels/test_acquire.py create mode 100644 tests/unit/seed/wheels/test_acquire_find_wheel.py create mode 100644 tests/unit/seed/wheels/test_periodic_update.py create mode 100644 tests/unit/seed/wheels/test_wheels_util.py diff --git a/.coveragerc b/.coveragerc index 000f9a560..2b79b2626 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,4 @@ [coverage:report] -skip_covered = True show_missing = True exclude_lines = \#\s*pragma: no cover @@ -9,8 +8,10 @@ exclude_lines = ^if __name__ == ['"]__main__['"]:$ omit = # site.py is ran before the coverage can be enabled, no way to measure coverage on this - src/virtualenv/interpreters/create/impl/cpython/site.py - src/virtualenv/seed/embed/wheels/pip-*.whl/* + src/virtualenv/create/via_global_ref/builtin/python2/site.py + src/virtualenv/create/via_global_ref/_virtualenv.py + src/virtualenv/activation/python/activate_this.py + src/virtualenv/seed/wheels/embed/pip-*.whl/* [coverage:paths] source = @@ -24,5 +25,9 @@ source = [coverage:run] branch = false parallel = true +dynamic_context = test_function source = ${_COVERAGE_SRC} + +[coverage:html] +show_contexts = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 921d5e8a6..63284f7c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.1.0 hooks: - id: check-ast - id: check-builtin-literals @@ -15,16 +15,12 @@ repos: rev: v2.0.1 hooks: - id: add-trailing-comma -- repo: https://github.com/asottile/yesqa - rev: v1.1.0 - hooks: - - id: yesqa - repo: https://github.com/asottile/pyupgrade - rev: v2.4.1 + rev: v2.6.1 hooks: - id: pyupgrade - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.1 + rev: v2.2.0 hooks: - id: seed-isort-config args: [--application-directories, '.:src'] @@ -54,8 +50,8 @@ repos: - id: setup-cfg-fmt args: [--min-py3-version, "3.4"] - repo: https://gitlab.com/pycqa/flake8 - rev: "3.8.1" + rev: "3.8.3" hooks: - id: flake8 - additional_dependencies: ["flake8-bugbear == 20.1.2"] + additional_dependencies: ["flake8-bugbear == 20.1.4"] language_version: python3.8 diff --git a/codecov.yaml b/codecov.yaml index 8df1c8e71..217805ed1 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -1,3 +1,8 @@ +ignore: +- "src/virtualenv/create/via_global_ref/builtin/python2/site.py" +- "src/virtualenv/create/via_global_ref/_virtualenv.py" +- "src/virtualenv/activation/python/activate_this.py" + coverage: range: "80...100" coverage: diff --git a/docs/changelog/1821.doc.rst b/docs/changelog/1821.doc.rst new file mode 100644 index 000000000..43e7e577a --- /dev/null +++ b/docs/changelog/1821.doc.rst @@ -0,0 +1 @@ +Document how bundled wheels are handled and (potentially automatically) kept up to date - by :user:`gaborbernat`. diff --git a/docs/changelog/1821.feature.rst b/docs/changelog/1821.feature.rst new file mode 100644 index 000000000..08070b8b0 --- /dev/null +++ b/docs/changelog/1821.feature.rst @@ -0,0 +1,8 @@ +Better handling of bundled wheel installation: + +- display the installed seed package versions in the final summary output +- add a manual upgrade of embedded wheels feature via :option:`upgrade-embed-wheels` CLI flag +- periodically (once every 14 days) try to automatically upgrade the embedded wheels in the background, can be disabled + via :option:`no-periodic-update` + +by :user:`gaborbernat`. diff --git a/docs/changelog/1841.feature.rst b/docs/changelog/1841.feature.rst new file mode 100644 index 000000000..38d808178 --- /dev/null +++ b/docs/changelog/1841.feature.rst @@ -0,0 +1,7 @@ +Bump embed wheel content: + +- ship wheels for Python ``3.9`` +- upgrade embedded setuptools for Python ``3.5+`` from ``46.4.0`` to ``47.1.1`` +- upgrade embedded setuptools for Python ``2.7`` from ``44.1.0`` to ``44.1.1`` + +by :user:`gaborbernat`. diff --git a/docs/cli_interface.rst b/docs/cli_interface.rst index db5da6ca5..f1c7b7e78 100644 --- a/docs/cli_interface.rst +++ b/docs/cli_interface.rst @@ -22,7 +22,7 @@ The options that can be passed to virtualenv, along with their default values an .. table_cli:: :module: virtualenv.run - :func: build_parser + :func: build_parser_only Defaults ~~~~~~~~ diff --git a/docs/user_guide.rst b/docs/user_guide.rst index eefbaf760..3dee8067a 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -120,7 +120,7 @@ enables you to install additional python packages into the created virtual envir main seed mechanism available: - ``pip`` - this method uses the bundled pip with virtualenv to install the seed packages (note, a new child process - needs to be created to do this). + needs to be created to do this, which can be expensive especially on Windows). - ``app-data`` - this method uses the user application data directory to create install images. These images are needed to be created only once, and subsequent virtual environments can just link/copy those images into their pure python library path (the ``site-packages`` folder). This allows all but the first virtual environment creation to be blazing @@ -131,6 +131,66 @@ main seed mechanism available: To override the filesystem location of the seed cache, one can use the ``VIRTUALENV_OVERRIDE_APP_DATA`` environment variable. +Wheels +~~~~~~ + +To install a seed package via either ``pip`` or ``app-data`` method virtualenv needs to acquire a wheel of the target +package. These wheels may be acquired from multiple locations as follows: + +- ``virtualenv`` ships out of box with a set of embed ``wheels`` for all three seed packages (:pypi:`pip`, + :pypi:`setuptools`, :pypi:`wheel`). These are packaged together with the virtualenv source files, and only change upon + upgrading virtualenv. Different Python versions require different versions of these, and because virtualenv supports a + wide range of Python versions, the number of embedded wheels out of box is greater than 3. Whenever newer versions of + these embedded packages are released upstream ``virtualenv`` project upgrades them, and does a new release. Therefore, + upgrading virtualenv periodically will also upgrade the version of the seed packages. +- However, end users might not be able to upgrade virtualenv at the same speed as we do new releases. Therefore, a user + might request to upgrade the list of embedded wheels by invoking virtualenv with the :option:`upgrade-embed-wheels` + flag. If the operation is triggered in such manual way subsequent runs of virtualenv will always use the upgraded + embed wheels. + + The operation can trigger automatically too, as a background process upon invocation of virtualenv, if no such upgrade + has been performed in the last 14 days. It will only start using automatically upgraded wheel if they have been + released for more than 28 days, and the automatic upgrade finished at least an hour ago: + + - the 28 days period should guarantee end users are not pulling in automatically releases that have known bugs within, + - the one hour period after the automatic upgrade finished is implemented so that continuous integration services do + not start using a new embedded versions half way through. + + + The automatic behaviour might be disabled via the :option:`no-periodic-update` configuration flag/option. To acquire + the release date of a package virtualenv will perform the following: + + - lookup ``https://pypi.org/pypi//json`` (primary truth source), + - save the date the version was first discovered, and wait until 28 days passed. +- Users can specify a set of local paths containing additional wheels by using the :option:`extra-search-dir` command + line argument flag. + +When searching for a wheel to use virtualenv performs lookup in the following order: + +- embedded wheels, +- upgraded embedded wheels, +- extra search dir. + +Bundled wheels are all three above together. If neither of the locations contain the requested wheel version or +:option:`download` option is set will use ``pip`` download to load the latest version available from the index server. + +Embed wheels for distributions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Custom distributions often want to use their own set of wheel versions to distribute instead of the one virtualenv +releases on PyPi. The reason for this is trying to keep the system versions of those package in sync with what +virtualenv uses. In such cases they should patch the module `virtualenv.seed.wheels.embed +`_, making sure to provide the function +``get_embed_wheel`` (which returns the wheel to use given a distribution/python version). The ``BUNDLE_FOLDER``, +``BUNDLE_SUPPORT`` and ``MAX`` variables are needed if they want to use virtualenvs test suite to validate. + +Furthermore, they might want to disable the periodic update by patching the +`virtualenv.seed.embed.base_embed.PERIODIC_UPDATE_ON_BY_DEFAULT +`_ +to ``False``, and letting the system update mechanism to handle this. Note in this case the user might still request an +upgrade of the embedded wheels by invoking virtualenv via :option:`upgrade-embed-wheels`, but no longer happens +automatically, and will not alter the OS provided wheels. + Activators ---------- These are activation scripts that will mangle with your shells settings to ensure that commands from within the python @@ -201,7 +261,7 @@ about the created virtual environment. .. automodule:: virtualenv :members: -.. currentmodule:: virtualenv.session +.. currentmodule:: virtualenv.run.session .. autoclass:: Session :members: diff --git a/setup.cfg b/setup.cfg index f83160644..a64a2c3db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,7 +43,6 @@ install_requires = distlib>=0.3.0,<1 filelock>=3.0.0,<4 six>=1.9.0,<2 # keep it >=1.9.0 as it may cause problems on LTS platforms - contextlib2>=0.6.0,<1;python_version<"3.3" importlib-metadata>=0.12,<2;python_version<"3.8" importlib-resources>=1.0;python_version<"3.7" pathlib2>=2.3.3,<3;python_version < '3.4' and sys.platform != 'win32' @@ -79,7 +78,7 @@ virtualenv.discovery = builtin = virtualenv.discovery.builtin:Builtin virtualenv.seed = pip = virtualenv.seed.embed.pip_invoke:PipInvoke - app-data = virtualenv.seed.via_app_data.via_app_data:FromAppData + app-data = virtualenv.seed.embed.via_app_data.via_app_data:FromAppData [options.extras_require] docs = @@ -98,6 +97,7 @@ testing = pytest-env >= 0.6.2 pytest-randomly >= 1 pytest-timeout >= 1 + pytest-freezegun >= 0.4.1 flaky >= 3 xonsh >= 0.9.16; python_version > '3.4' and python_version != '3.9' @@ -108,7 +108,7 @@ virtualenv.activation.cshell = *.csh virtualenv.activation.fish = *.fish virtualenv.activation.powershell = *.ps1 virtualenv.activation.xonsh = *.xsh -virtualenv.seed.embed.wheels = *.whl +virtualenv.seed.wheels.embed = *.whl [options.packages.find] where = src diff --git a/setup.py b/setup.py index 0f0ab7a9f..8ba6e3778 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,9 @@ raise RuntimeError("setuptools >= 41 required to build") setup( - use_scm_version={"write_to": "src/virtualenv/version.py", "write_to_template": '__version__ = "{version}"'}, + use_scm_version={ + "write_to": "src/virtualenv/version.py", + "write_to_template": 'from __future__ import unicode_literals;\n\n__version__ = "{version}"', + }, setup_requires=["setuptools_scm >= 2"], ) diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py index 5d47db8cd..c87fc4147 100644 --- a/src/virtualenv/__main__.py +++ b/src/virtualenv/__main__.py @@ -5,13 +5,10 @@ import sys from datetime import datetime -from virtualenv.config.cli.parser import VirtualEnvOptions -from virtualenv.util.six import ensure_text - def run(args=None, options=None): start = datetime.now() - from virtualenv.error import ProcessCallFailed + from virtualenv.util.error import ProcessCallFailed from virtualenv.run import cli_run if args is None: @@ -32,6 +29,8 @@ def __init__(self, session, start): self.start = start def __str__(self): + from virtualenv.util.six import ensure_text + spec = self.session.creator.interpreter.spec elapsed = (datetime.now() - self.start).total_seconds() * 1000 lines = [ @@ -39,24 +38,40 @@ def __str__(self): " creator {}".format(ensure_text(str(self.session.creator))), ] if self.session.seeder.enabled: - lines += (" seeder {}".format(ensure_text(str(self.session.seeder))),) + lines += ( + " seeder {}".format(ensure_text(str(self.session.seeder))), + " added seed packages: {}".format( + ", ".join( + sorted( + "==".join(i.stem.split("-")) + for i in self.session.creator.purelib.iterdir() + if i.suffix == ".dist-info" + ), + ), + ), + ) if self.session.activators: lines.append(" activators {}".format(",".join(i.__class__.__name__ for i in self.session.activators))) return os.linesep.join(lines) def run_with_catch(args=None): + from virtualenv.config.cli.parser import VirtualEnvOptions + options = VirtualEnvOptions() try: run(args, options) - except (KeyboardInterrupt, Exception) as exception: - if getattr(options, "with_traceback", False): + except (KeyboardInterrupt, SystemExit, Exception) as exception: + try: + if getattr(options, "with_traceback", False): + raise + else: + logging.error("%s: %s", type(exception).__name__, exception) + code = exception.code if isinstance(exception, SystemExit) else 1 + sys.exit(code) + finally: logging.shutdown() # force flush of log messages before the trace is printed - raise - else: - logging.error("%s: %s", type(exception).__name__, exception) - sys.exit(1) -if __name__ == "__main__": - run_with_catch() +if __name__ == "__main__": # pragma: no cov + run_with_catch() # pragma: no cov diff --git a/src/virtualenv/run/app_data.py b/src/virtualenv/app_data/__init__.py similarity index 59% rename from src/virtualenv/run/app_data.py rename to src/virtualenv/app_data/__init__.py index 68edb93f1..1d85745a4 100644 --- a/src/virtualenv/run/app_data.py +++ b/src/virtualenv/app_data/__init__.py @@ -1,39 +1,17 @@ +""" +Application data stored by virtualenv. +""" +from __future__ import absolute_import, unicode_literals + import logging import os from argparse import Action, ArgumentError -from tempfile import mkdtemp from appdirs import user_data_dir -from virtualenv.util.lock import ReentrantFileLock -from virtualenv.util.path import safe_delete - - -class AppData(object): - def __init__(self, folder): - self.folder = ReentrantFileLock(folder) - self.transient = False - - def __repr__(self): - return "{}".format(self.folder.path) - - def clean(self): - logging.debug("clean app data folder %s", self.folder.path) - safe_delete(self.folder.path) - - def close(self): - """""" - - -class TempAppData(AppData): - def __init__(self): - super(TempAppData, self).__init__(folder=mkdtemp()) - self.transient = True - logging.debug("created temporary app data folder %s", self.folder.path) - - def close(self): - logging.debug("remove temporary app data folder %s", self.folder.path) - safe_delete(self.folder.path) +from .na import AppDataDisabled +from .via_disk_folder import AppDataDiskFolder +from .via_tempdir import TempAppData class AppDataAction(Action): @@ -41,7 +19,7 @@ def __call__(self, parser, namespace, values, option_string=None): folder = self._check_folder(values) if folder is None: raise ArgumentError("app data path {} is not valid".format(values)) - setattr(namespace, self.dest, AppData(folder)) + setattr(namespace, self.dest, AppDataDiskFolder(folder)) @staticmethod def _check_folder(folder): @@ -64,8 +42,8 @@ def default(): for folder in AppDataAction._app_data_candidates(): folder = AppDataAction._check_folder(folder) if folder is not None: - return AppData(folder) - return None + return AppDataDiskFolder(folder) + return AppDataDisabled() @staticmethod def _app_data_candidates(): @@ -77,7 +55,8 @@ def _app_data_candidates(): __all__ = ( - "AppData", + "AppDataDiskFolder", "TempAppData", "AppDataAction", + "AppDataDisabled", ) diff --git a/src/virtualenv/app_data/base.py b/src/virtualenv/app_data/base.py new file mode 100644 index 000000000..d0da0fc11 --- /dev/null +++ b/src/virtualenv/app_data/base.py @@ -0,0 +1,91 @@ +""" +Application data stored by virtualenv. +""" +from __future__ import absolute_import, unicode_literals + +from abc import ABCMeta, abstractmethod +from contextlib import contextmanager + +import six + +from virtualenv.info import IS_ZIPAPP + + +@six.add_metaclass(ABCMeta) +class AppData(object): + """Abstract storage interface for the virtualenv application""" + + @abstractmethod + def close(self): + """called before virtualenv exits""" + + @abstractmethod + def reset(self): + """called when the user passes in the reset app data""" + + @abstractmethod + def py_info(self, path): + raise NotImplementedError + + @abstractmethod + def py_info_clear(self): + raise NotImplementedError + + @abstractmethod + def embed_update_log(self, distribution, for_py_version): + raise NotImplementedError + + @property + def house(self): + raise NotImplementedError + + @property + def transient(self): + raise NotImplementedError + + @abstractmethod + def wheel_image(self, for_py_version, name): + raise NotImplementedError + + @contextmanager + def ensure_extracted(self, path, to_folder=None): + """Some paths might be within the zipapp, unzip these to a path on the disk""" + if IS_ZIPAPP: + with self.extract(path, to_folder) as result: + yield result + else: + yield path + + @abstractmethod + @contextmanager + def extract(self, path, to_folder): + raise NotImplementedError + + @abstractmethod + @contextmanager + def locked(self, path): + raise NotImplementedError + + +@six.add_metaclass(ABCMeta) +class ContentStore(object): + @abstractmethod + def exists(self): + raise NotImplementedError + + @abstractmethod + def read(self): + raise NotImplementedError + + @abstractmethod + def write(self, content): + raise NotImplementedError + + @abstractmethod + def remove(self): + raise NotImplementedError + + @abstractmethod + @contextmanager + def locked(self): + pass diff --git a/src/virtualenv/app_data/na.py b/src/virtualenv/app_data/na.py new file mode 100644 index 000000000..937aa9a47 --- /dev/null +++ b/src/virtualenv/app_data/na.py @@ -0,0 +1,67 @@ +from __future__ import absolute_import, unicode_literals + +from contextlib import contextmanager + +from .base import AppData, ContentStore + + +class AppDataDisabled(AppData): + """No application cache available (most likely as we don't have write permissions)""" + + def __init__(self): + pass + + error = RuntimeError("no app data folder available, probably no write access to the folder") + + def close(self): + """do nothing""" + + def reset(self): + """do nothing""" + + def py_info(self, path): + return ContentStoreNA() + + def embed_update_log(self, distribution, for_py_version): + return ContentStoreNA() + + def extract(self, path, to_folder): + raise self.error + + @contextmanager + def locked(self, path): + """do nothing""" + yield + + @property + def house(self): + raise self.error + + def wheel_image(self, for_py_version, name): + raise self.error + + @property + def transient(self): + return True + + def py_info_clear(self): + """""" + + +class ContentStoreNA(ContentStore): + def exists(self): + return False + + def read(self): + """""" + return None + + def write(self, content): + """""" + + def remove(self): + """""" + + @contextmanager + def locked(self): + yield diff --git a/src/virtualenv/app_data/via_disk_folder.py b/src/virtualenv/app_data/via_disk_folder.py new file mode 100644 index 000000000..6b12ef8b0 --- /dev/null +++ b/src/virtualenv/app_data/via_disk_folder.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +A rough layout of the current storage goes as: + +virtualenv-app-data +├── py - +│   └── *.json/lock +├── wheel +│   ├── house +│ │ └── *.whl +│ └── -> 3.9 +│ ├── img- +│ │ └── image +│ │ └── -> CopyPipInstall / SymlinkPipInstall +│ │ └── -> pip-20.1.1-py2.py3-none-any +│ └── embed +│ └── 1 +│ └── *.json -> for every distribution contains data about newer embed versions and releases +└─── unzip + └── + ├── py_info.py + ├── debug.py + └── _virtualenv.py +""" +from __future__ import absolute_import, unicode_literals + +import json +import logging +from abc import ABCMeta +from contextlib import contextmanager +from hashlib import sha256 + +import six + +from virtualenv.util.lock import ReentrantFileLock +from virtualenv.util.path import safe_delete +from virtualenv.util.six import ensure_text +from virtualenv.util.zipapp import extract +from virtualenv.version import __version__ + +from .base import AppData, ContentStore + + +class AppDataDiskFolder(AppData): + """ + Store the application data on the disk within a folder layout. + """ + + def __init__(self, folder): + self.lock = ReentrantFileLock(folder) + + def __repr__(self): + return "{}".format(self.lock.path) + + @property + def transient(self): + return False + + def reset(self): + logging.debug("reset app data folder %s", self.lock.path) + safe_delete(self.lock.path) + + def close(self): + """do nothing""" + + @contextmanager + def locked(self, path): + path_lock = self.lock / path + with path_lock: + yield path_lock.path + + @contextmanager + def extract(self, path, to_folder): + if to_folder is not None: + root = ReentrantFileLock(to_folder()) + else: + root = self.lock / "unzip" / __version__ + with root.lock_for_key(path.name): + dest = root.path / path.name + if not dest.exists(): + extract(path, dest) + yield dest + + @property + def py_info_at(self): + return self.lock / "py_info" / "1" + + def py_info(self, path): + return PyInfoStoreDisk(self.py_info_at, path) + + def py_info_clear(self): + """""" + py_info_folder = self.py_info_at + with py_info_folder: + for filename in py_info_folder.path.iterdir(): + if filename.suffix == ".json": + with py_info_folder.lock_for_key(filename.stem): + if filename.exists(): + filename.unlink() + + def embed_update_log(self, distribution, for_py_version): + return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "1", distribution) + + @property + def house(self): + path = self.lock.path / "wheel" / "house" + path.mkdir(parents=True, exist_ok=True) + return path + + def wheel_image(self, for_py_version, name): + return self.lock.path / "wheel" / for_py_version / "image" / "1" / name + + +@six.add_metaclass(ABCMeta) +class JSONStoreDisk(ContentStore): + def __init__(self, in_folder, key, msg, msg_args): + self.in_folder = in_folder + self.key = key + self.msg = msg + self.msg_args = msg_args + (self.file,) + + @property + def file(self): + return self.in_folder.path / "{}.json".format(self.key) + + def exists(self): + return self.file.exists() + + def read(self): + data, bad_format = None, False + try: + data = json.loads(self.file.read_text()) + logging.debug("got {} from %s".format(self.msg), *self.msg_args) + return data + except ValueError: + bad_format = True + except Exception: # noqa + pass + if bad_format: + self.remove() + return None + + def remove(self): + self.file.unlink() + logging.debug("removed {} at %s".format(self.msg), *self.msg_args) + + @contextmanager + def locked(self): + with self.in_folder.lock_for_key(self.key): + yield + + def write(self, content): + folder = self.file.parent + try: + folder.mkdir(parents=True, exist_ok=True) + except OSError: + pass + self.file.write_text(ensure_text(json.dumps(content, sort_keys=True, indent=2))) + logging.debug("wrote {} at %s".format(self.msg), *self.msg_args) + + +class PyInfoStoreDisk(JSONStoreDisk): + def __init__(self, in_folder, path): + key = sha256(str(path).encode("utf-8") if six.PY3 else str(path)).hexdigest() + super(PyInfoStoreDisk, self).__init__(in_folder, key, "python info of %s", (path,)) + + +class EmbedDistributionUpdateStoreDisk(JSONStoreDisk): + def __init__(self, in_folder, distribution): + super(EmbedDistributionUpdateStoreDisk, self).__init__( + in_folder, distribution, "embed update of distribution %s", (distribution,), + ) diff --git a/src/virtualenv/app_data/via_tempdir.py b/src/virtualenv/app_data/via_tempdir.py new file mode 100644 index 000000000..e8b387c52 --- /dev/null +++ b/src/virtualenv/app_data/via_tempdir.py @@ -0,0 +1,28 @@ +from __future__ import absolute_import, unicode_literals + +import logging +from tempfile import mkdtemp + +from virtualenv.util.path import safe_delete + +from .via_disk_folder import AppDataDiskFolder + + +class TempAppData(AppDataDiskFolder): + def __init__(self): + super(TempAppData, self).__init__(folder=mkdtemp()) + logging.debug("created temporary app data folder %s", self.lock.path) + + def reset(self): + """this is a temporary folder, is already empty to start with""" + + def close(self): + logging.debug("remove temporary app data folder %s", self.lock.path) + safe_delete(self.lock.path) + + def embed_update_log(self, distribution, for_py_version): + return None + + @property + def transient(self): + return True diff --git a/src/virtualenv/config/cli/parser.py b/src/virtualenv/config/cli/parser.py index 1c4d62c7b..eb4db30a7 100644 --- a/src/virtualenv/config/cli/parser.py +++ b/src/virtualenv/config/cli/parser.py @@ -56,7 +56,6 @@ def __init__(self, options=None, *args, **kwargs): kwargs["prog"] = "virtualenv" super(VirtualEnvConfigParser, self).__init__(*args, **kwargs) self._fixed = set() - self._elements = None if options is not None and not isinstance(options, VirtualEnvOptions): raise TypeError("options must be of type VirtualEnvOptions") self.options = VirtualEnvOptions() if options is None else options diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py index 81fcd157b..0dcc49cda 100644 --- a/src/virtualenv/create/creator.py +++ b/src/virtualenv/create/creator.py @@ -14,13 +14,13 @@ from virtualenv.discovery.cached_py_info import LogCmd from virtualenv.info import WIN_CPYTHON_2 -from virtualenv.pyenv_cfg import PyEnvCfg from virtualenv.util.path import Path, safe_delete from virtualenv.util.six import ensure_str, ensure_text from virtualenv.util.subprocess import run_cmd -from virtualenv.util.zipapp import ensure_file_on_disk from virtualenv.version import __version__ +from .pyenv_cfg import PyEnvCfg + HERE = Path(os.path.abspath(__file__)).parent DEBUG_SCRIPT = HERE / "debug.py" @@ -45,7 +45,7 @@ def __init__(self, options, interpreter): self.dest = Path(options.dest) self.clear = options.clear self.pyenv_cfg = PyEnvCfg.from_folder(self.dest) - self.app_data = options.app_data.folder + self.app_data = options.app_data def __repr__(self): return ensure_str(self.__unicode__()) @@ -74,6 +74,7 @@ def add_parser_arguments(cls, parser, interpreter, meta, app_data): """Add CLI arguments for the creator. :param parser: the CLI parser + :param app_data: the application data folder :param interpreter: the interpreter we're asked to create virtual environment for :param meta: value as returned by :meth:`can_create` """ @@ -199,7 +200,7 @@ def get_env_debug_info(env_exe, debug_script, app_data): env = os.environ.copy() env.pop(str("PYTHONPATH"), None) - with ensure_file_on_disk(debug_script, app_data) as debug_script: + with app_data.ensure_extracted(debug_script) as debug_script: cmd = [str(env_exe), str(debug_script)] if WIN_CPYTHON_2: cmd = [ensure_text(i) for i in cmd] diff --git a/src/virtualenv/pyenv_cfg.py b/src/virtualenv/create/pyenv_cfg.py similarity index 100% rename from src/virtualenv/pyenv_cfg.py rename to src/virtualenv/create/pyenv_cfg.py diff --git a/src/virtualenv/create/via_global_ref/api.py b/src/virtualenv/create/via_global_ref/api.py index 412e871a4..c9eab3c6e 100644 --- a/src/virtualenv/create/via_global_ref/api.py +++ b/src/virtualenv/create/via_global_ref/api.py @@ -8,7 +8,6 @@ from virtualenv.info import fs_supports_symlink from virtualenv.util.path import Path -from virtualenv.util.zipapp import ensure_file_on_disk from ..creator import Creator, CreatorMeta @@ -100,7 +99,7 @@ def install_patch(self): def env_patch_text(self): """Patch the distutils package to not be derailed by its configuration files""" - with ensure_file_on_disk(Path(__file__).parent / "_virtualenv.py", self.app_data) as resolved_path: + with self.app_data.ensure_extracted(Path(__file__).parent / "_virtualenv.py") as resolved_path: text = resolved_path.read_text() return text.replace('"__SCRIPT_DIR__"', repr(os.path.relpath(str(self.script_dir), str(self.purelib)))) diff --git a/src/virtualenv/create/via_global_ref/store.py b/src/virtualenv/create/via_global_ref/store.py index 55d941347..134a53585 100644 --- a/src/virtualenv/create/via_global_ref/store.py +++ b/src/virtualenv/create/via_global_ref/store.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, unicode_literals + from virtualenv.util.path import Path @@ -18,4 +20,7 @@ def is_store_python(interpreter): ) -__all__ = ("handle_store_python", "is_store_python") +__all__ = ( + "handle_store_python", + "is_store_python", +) diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py index 88e74e36a..4a4ed7701 100644 --- a/src/virtualenv/create/via_global_ref/venv.py +++ b/src/virtualenv/create/via_global_ref/venv.py @@ -5,7 +5,7 @@ from virtualenv.create.via_global_ref.store import handle_store_python from virtualenv.discovery.py_info import PythonInfo -from virtualenv.error import ProcessCallFailed +from virtualenv.util.error import ProcessCallFailed from virtualenv.util.path import ensure_dir from virtualenv.util.subprocess import run_cmd diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index 24985555a..4d57fa581 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -30,7 +30,7 @@ def add_parser_arguments(cls, parser): ) def run(self): - return get_interpreter(self.python_spec, self.app_data.folder) + return get_interpreter(self.python_spec, self.app_data) def __repr__(self): return ensure_str(self.__unicode__()) diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index a1fd3f3ef..13a213d76 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -6,21 +6,18 @@ """ from __future__ import absolute_import, unicode_literals -import json import logging import os import pipes import sys from collections import OrderedDict -from hashlib import sha256 +from virtualenv.app_data import AppDataDisabled from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import PY2, PY3 +from virtualenv.info import PY2 from virtualenv.util.path import Path from virtualenv.util.six import ensure_text from virtualenv.util.subprocess import Popen, subprocess -from virtualenv.util.zipapp import ensure_file_on_disk -from virtualenv.version import __version__ _CACHE = OrderedDict() _CACHE[Path(sys.executable)] = PythonInfo() @@ -28,8 +25,7 @@ def from_exe(cls, app_data, exe, raise_on_error=True, ignore_cache=False): """""" - py_info_cache = _get_py_info_cache(app_data) - result = _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=ignore_cache) + result = _get_from_cache(cls, app_data, exe, ignore_cache=ignore_cache) if isinstance(result, Exception): if raise_on_error: raise result @@ -39,21 +35,14 @@ def from_exe(cls, app_data, exe, raise_on_error=True, ignore_cache=False): return result -def _get_py_info_cache(app_data): - return None if app_data is None else app_data / "py_info" / __version__ - - -def _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=True): +def _get_from_cache(cls, app_data, exe, ignore_cache=True): # note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a # pyenv.cfg somewhere alongside on python3.4+ exe_path = Path(exe) if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache result = _CACHE[exe_path] - elif py_info_cache is None: # cache disabled - failure, py_info = _run_subprocess(cls, exe, app_data) - result = py_info if failure is None else failure - else: # then check the persisted cache - py_info = _get_via_file_cache(cls, py_info_cache, app_data, exe_path, exe) + else: # otherwise go through the app data cache + py_info = _get_via_file_cache(cls, app_data, exe_path, exe) result = _CACHE[exe_path] = py_info # independent if it was from the file or in-memory cache fix the original executable location if isinstance(result, PythonInfo): @@ -61,47 +50,37 @@ def _get_from_cache(cls, py_info_cache, app_data, exe, ignore_cache=True): return result -def _get_via_file_cache(cls, py_info_cache, app_data, resolved_path, exe): - key = sha256(str(resolved_path).encode("utf-8") if PY3 else str(resolved_path)).hexdigest() - py_info = None - resolved_path_text = ensure_text(str(resolved_path)) +def _get_via_file_cache(cls, app_data, path, exe): + path_text = ensure_text(str(path)) try: - resolved_path_modified_timestamp = resolved_path.stat().st_mtime + path_modified = path.stat().st_mtime except OSError: - resolved_path_modified_timestamp = -1 - data_file = py_info_cache / "{}.json".format(key) - with py_info_cache.lock_for_key(key): - data_file_path = data_file.path - if data_file_path.exists() and resolved_path_modified_timestamp != 1: # if exists and matches load - try: - data = json.loads(data_file_path.read_text()) - if data["path"] == resolved_path_text and data["st_mtime"] == resolved_path_modified_timestamp: - logging.debug("get PythonInfo from %s for %s", data_file_path, exe) - py_info = cls._from_dict({k: v for k, v in data["content"].items()}) - else: - raise ValueError("force close as stale") - except (KeyError, ValueError, OSError): - logging.debug("remove PythonInfo %s for %s", data_file_path, exe) - data_file_path.unlink() # close out of date files + path_modified = -1 + if app_data is None: + app_data = AppDataDisabled() + py_info, py_info_store = None, app_data.py_info(path) + with py_info_store.locked(): + if py_info_store.exists(): # if exists and matches load + data = py_info_store.read() + of_path, of_st_mtime, of_content = data["path"], data["st_mtime"], data["content"] + if of_path == path_text and of_st_mtime == path_modified: + py_info = cls._from_dict({k: v for k, v in of_content.items()}) + else: + py_info_store.remove() if py_info is None: # if not loaded run and save failure, py_info = _run_subprocess(cls, exe, app_data) if failure is None: - file_cache_content = { - "st_mtime": resolved_path_modified_timestamp, - "path": resolved_path_text, - "content": py_info._to_dict(), - } - logging.debug("write PythonInfo to %s for %s", data_file_path, exe) - data_file_path.write_text(ensure_text(json.dumps(file_cache_content, indent=2))) + data = {"st_mtime": path_modified, "path": path_text, "content": py_info._to_dict()} + py_info_store.write(data) else: py_info = failure return py_info def _run_subprocess(cls, exe, app_data): - resolved_path = Path(os.path.abspath(__file__)).parent / "py_info.py" - with ensure_file_on_disk(resolved_path, app_data) as resolved_path: - cmd = [exe, str(resolved_path)] + py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" + with app_data.ensure_extracted(py_info_script) as py_info_script: + cmd = [exe, str(py_info_script)] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = os.environ.copy() env.pop("__PYVENV_LAUNCHER__", None) @@ -155,14 +134,7 @@ def __unicode__(self): def clear(app_data): - py_info_cache = _get_py_info_cache(app_data) - if py_info_cache is not None: - with py_info_cache: - for filename in py_info_cache.path.iterdir(): - if filename.suffix == ".json": - with py_info_cache.lock_for_key(filename.stem): - if filename.exists(): - filename.unlink() + app_data.py_info_clear() _CACHE.clear() diff --git a/src/virtualenv/report.py b/src/virtualenv/report.py index 7ae2f2414..8d5c35871 100644 --- a/src/virtualenv/report.py +++ b/src/virtualenv/report.py @@ -40,6 +40,7 @@ def setup_report(verbosity): LOGGER.addHandler(stream_handler) level_name = logging.getLevelName(level) logging.debug("setup logging to %s", level_name) + logging.getLogger("distlib").setLevel(logging.ERROR) return verbosity diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index ed6d808f3..8de7962af 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -2,11 +2,11 @@ import logging -from virtualenv.run.app_data import AppDataAction - +from ..app_data import AppDataAction, AppDataDisabled, TempAppData from ..config.cli.parser import VirtualEnvConfigParser from ..report import LEVELS, setup_report -from ..session import Session +from ..run.session import Session +from ..seed.wheels.periodic_update import manual_upgrade from ..version import __version__ from .plugin.activators import ActivationSelector from .plugin.creators import CreatorSelector @@ -29,9 +29,9 @@ def cli_run(args, options=None): # noinspection PyProtectedMember def session_via_cli(args, options=None): - parser = build_parser(args, options) + parser, elements = build_parser(args, options) options = parser.parse_args(args) - creator, seeder, activators = tuple(e.create(options) for e in parser._elements) # create types + creator, seeder, activators = tuple(e.create(options) for e in elements) # create types session = Session(options.verbosity, options.app_data, parser._interpreter, creator, seeder, activators) return session @@ -48,13 +48,44 @@ def build_parser(args=None, options=None): help="on failure also display the stacktrace internals of virtualenv", ) _do_report_setup(parser, args) + options = load_app_data(args, parser, options) + handle_extra_commands(options) + + discover = get_discover(parser, args) + parser._interpreter = interpreter = discover.interpreter + if interpreter is None: + raise RuntimeError("failed to find interpreter for {}".format(discover)) + elements = [ + CreatorSelector(interpreter, parser), + SeederSelector(interpreter, parser), + ActivationSelector(interpreter, parser), + ] + options, _ = parser.parse_known_args(args) + for element in elements: + element.handle_selected_arg_parse(options) + parser.enable_help() + return parser, elements + + +def build_parser_only(args=None): + """Used to provide a parser for the doc generation""" + return build_parser(args)[0] + + +def handle_extra_commands(options): + if options.upgrade_embed_wheels: + result = manual_upgrade(options.app_data) + raise SystemExit(result) + + +def load_app_data(args, parser, options): # here we need a write-able application data (e.g. the zipapp might need this for discovery cache) default_app_data = AppDataAction.default() parser.add_argument( "--app-data", dest="app_data", action=AppDataAction, - default="" if default_app_data is None else default_app_data, + default="" if isinstance(default_app_data, AppDataDisabled) else default_app_data, help="a data folder used as cache by the virtualenv", ) parser.add_argument( @@ -64,20 +95,19 @@ def build_parser(args=None, options=None): help="start with empty app data folder", default=False, ) - discover = get_discover(parser, args) - parser._interpreter = interpreter = discover.interpreter - if interpreter is None: - raise RuntimeError("failed to find interpreter for {}".format(discover)) - parser._elements = [ - CreatorSelector(interpreter, parser), - SeederSelector(interpreter, parser), - ActivationSelector(interpreter, parser), - ] - options, _ = parser.parse_known_args(args) - for element in parser._elements: - element.handle_selected_arg_parse(options) - parser.enable_help() - return parser + parser.add_argument( + "--upgrade-embed-wheels", + dest="upgrade_embed_wheels", + action="store_true", + help="trigger a manual update of the embedded wheels", + default=False, + ) + options, _ = parser.parse_known_args(args, namespace=options) + if options.app_data == "": + options.app_data = TempAppData() + if options.reset_app_data: + options.app_data.reset() + return options def add_version_flag(parser): diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py index 9365e5e66..e2cfe9276 100644 --- a/src/virtualenv/run/plugin/discovery.py +++ b/src/virtualenv/run/plugin/discovery.py @@ -1,7 +1,5 @@ from __future__ import absolute_import, unicode_literals -from virtualenv.run.app_data import TempAppData - from .base import PluginLoader @@ -22,10 +20,6 @@ def get_discover(parser, args): help="interpreter discovery method", ) options, _ = parser.parse_known_args(args) - if options.app_data == "": - options.app_data = TempAppData() - if options.reset_app_data: - options.app_data.clean() discover_class = discover_types[options.discovery] discover_class.add_parser_arguments(discovery_parser) options, _ = parser.parse_known_args(args, namespace=options) diff --git a/src/virtualenv/session.py b/src/virtualenv/run/session.py similarity index 100% rename from src/virtualenv/session.py rename to src/virtualenv/run/session.py diff --git a/src/virtualenv/seed/embed/__init__.py b/src/virtualenv/seed/embed/__init__.py index 01e6d4f49..e69de29bb 100644 --- a/src/virtualenv/seed/embed/__init__.py +++ b/src/virtualenv/seed/embed/__init__.py @@ -1 +0,0 @@ -from __future__ import absolute_import, unicode_literals diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index bffd49478..f41b5fc49 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -8,38 +8,45 @@ from virtualenv.util.six import ensure_str, ensure_text from ..seeder import Seeder +from ..wheels import Version + +PERIODIC_UPDATE_ON_BY_DEFAULT = True @add_metaclass(ABCMeta) class BaseEmbed(Seeder): - packages = ["pip", "setuptools", "wheel"] - def __init__(self, options): super(BaseEmbed, self).__init__(options, enabled=options.no_seed is False) + self.download = options.download self.extra_search_dir = [i.resolve() for i in options.extra_search_dir if i.exists()] - def latest_is_none(key): - value = getattr(options, key) - return None if value == "latest" else value - - self.pip_version = latest_is_none("pip") - self.setuptools_version = latest_is_none("setuptools") - self.wheel_version = latest_is_none("wheel") + self.pip_version = options.pip + self.setuptools_version = options.setuptools + self.wheel_version = options.wheel self.no_pip = options.no_pip self.no_setuptools = options.no_setuptools self.no_wheel = options.no_wheel - self.app_data = options.app_data.folder + self.app_data = options.app_data + self.periodic_update = not options.no_periodic_update - if not self.package_version(): + if not self.distribution_to_versions(): self.enabled = False - def package_version(self): + @classmethod + def distributions(cls): return { - package: getattr(self, "{}_version".format(package)) - for package in self.packages - if getattr(self, "no_{}".format(package)) is False + "pip": Version.bundle, + "setuptools": Version.bundle, + "wheel": Version.bundle, + } + + def distribution_to_versions(self): + return { + distribution: getattr(self, "{}_version".format(distribution)) + for distribution in self.distributions() + if getattr(self, "no_{}".format(distribution)) is False } @classmethod @@ -50,14 +57,14 @@ def add_parser_arguments(cls, parser, interpreter, app_data): "--never-download", dest="download", action="store_false", - help="pass to disable download of the latest {} from PyPI".format("/".join(cls.packages)), + help="pass to disable download of the latest {} from PyPI".format("/".join(cls.distributions())), default=True, ) group.add_argument( "--download", dest="download", action="store_true", - help="pass to enable download of the latest {} from PyPI".format("/".join(cls.packages)), + help="pass to enable download of the latest {} from PyPI".format("/".join(cls.distributions())), default=False, ) parser.add_argument( @@ -65,25 +72,32 @@ def add_parser_arguments(cls, parser, interpreter, app_data): metavar="d", type=Path, nargs="+", - help="a path containing wheels the seeder may also use beside bundled (can be set 1+ times)", + help="a path containing wheels to extend the internal wheel list (can be set 1+ times)", default=[], ) - for package in cls.packages: + for distribution, default in cls.distributions().items(): parser.add_argument( - "--{}".format(package), - dest=package, + "--{}".format(distribution), + dest=distribution, metavar="version", - help="{} version to install, bundle for bundled".format(package), - default="latest", + help="version of {} to install as seed: embed, bundle or exact version".format(distribution), + default=default, ) - for package in cls.packages: + for distribution in cls.distributions(): parser.add_argument( - "--no-{}".format(package), - dest="no_{}".format(package), + "--no-{}".format(distribution), + dest="no_{}".format(distribution), action="store_true", - help="do not install {}".format(package), + help="do not install {}".format(distribution), default=False, ) + parser.add_argument( + "--no-periodic-update", + dest="no_periodic_update", + action="store_true", + help="disable the periodic (once every 14 days) update of the embedded wheels", + default=not PERIODIC_UPDATE_ON_BY_DEFAULT, + ) def __unicode__(self): result = self.__class__.__name__ @@ -91,11 +105,11 @@ def __unicode__(self): if self.extra_search_dir: result += "extra_search_dir={},".format(", ".join(ensure_text(str(i)) for i in self.extra_search_dir)) result += "download={},".format(self.download) - for package in self.packages: - if getattr(self, "no_{}".format(package)): + for distribution in self.distributions(): + if getattr(self, "no_{}".format(distribution)): continue result += " {}{},".format( - package, "={}".format(getattr(self, "{}_version".format(package), None) or "latest"), + distribution, "={}".format(getattr(self, "{}_version".format(distribution), None) or "latest"), ) return result[:-1] + ")" diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 25be493af..372e140dc 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -4,16 +4,10 @@ from contextlib import contextmanager from virtualenv.discovery.cached_py_info import LogCmd -from virtualenv.info import PY3 from virtualenv.seed.embed.base_embed import BaseEmbed -from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, pip_wheel_env_run from virtualenv.util.subprocess import Popen -from virtualenv.util.zipapp import ensure_file_on_disk -if PY3: - from contextlib import ExitStack -else: - from contextlib2 import ExitStack +from ..wheels import Version, get_wheel, pip_wheel_env_run class PipInvoke(BaseEmbed): @@ -23,9 +17,10 @@ def __init__(self, options): def run(self, creator): if not self.enabled: return - with self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) as cmd: - with pip_wheel_env_run(creator.interpreter.version_release_str, self.app_data) as env: - self._execute(cmd, env) + for_py_version = creator.interpreter.version_release_str + with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd: + env = pip_wheel_env_run(self.extra_search_dir, self.app_data) + self._execute(cmd, env) @staticmethod def _execute(cmd, env): @@ -37,18 +32,25 @@ def _execute(cmd, env): return process @contextmanager - def get_pip_install_cmd(self, exe, version): - cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:"] + def get_pip_install_cmd(self, exe, for_py_version): + cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:", "--disable-pip-version-check"] if not self.download: cmd.append("--no-index") - pkg_versions = self.package_version() - for key, ver in pkg_versions.items(): - cmd.append("{}{}".format(key, "=={}".format(ver) if ver is not None else "")) - with ExitStack() as stack: - folders = set() - for context in (ensure_file_on_disk(get_bundled_wheel(p, version), self.app_data) for p in pkg_versions): - folders.add(stack.enter_context(context).parent) - folders.update(set(self.extra_search_dir)) - for folder in folders: - cmd.extend(["--find-links", str(folder)]) - yield cmd + folders = set() + for dist, version in self.distribution_to_versions().items(): + wheel = get_wheel( + distribution=dist, + version=version, + for_py_version=for_py_version, + search_dirs=self.extra_search_dir, + download=False, + app_data=self.app_data, + do_periodic_update=self.periodic_update, + ) + if wheel is None: + raise RuntimeError("could not get wheel for distribution {}".format(dist)) + folders.add(str(wheel.path.parent)) + cmd.append(Version.as_pip_req(dist, wheel.version)) + for folder in sorted(folders): + cmd.extend(["--find-links", str(folder)]) + yield cmd diff --git a/src/virtualenv/seed/via_app_data/__init__.py b/src/virtualenv/seed/embed/via_app_data/__init__.py similarity index 100% rename from src/virtualenv/seed/via_app_data/__init__.py rename to src/virtualenv/seed/embed/via_app_data/__init__.py diff --git a/src/virtualenv/seed/via_app_data/pip_install/__init__.py b/src/virtualenv/seed/embed/via_app_data/pip_install/__init__.py similarity index 100% rename from src/virtualenv/seed/via_app_data/pip_install/__init__.py rename to src/virtualenv/seed/embed/via_app_data/pip_install/__init__.py diff --git a/src/virtualenv/seed/via_app_data/pip_install/base.py b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py similarity index 97% rename from src/virtualenv/seed/via_app_data/pip_install/base.py rename to src/virtualenv/seed/embed/via_app_data/pip_install/base.py index f7f29ca8e..f382bdacf 100644 --- a/src/virtualenv/seed/via_app_data/pip_install/base.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py @@ -54,11 +54,11 @@ def install(self, version_info): def build_image(self): # 1. first extract the wheel - logging.debug("build install image to %s of %s", self._image_dir, self._wheel.name) + logging.debug("build install image for %s to %s", self._wheel.name, self._image_dir) with zipfile.ZipFile(str(self._wheel)) as zip_ref: zip_ref.extractall(str(self._image_dir)) self._extracted = True - # 2. now add additional files not present in the package + # 2. now add additional files not present in the distribution new_files = self._generate_new_files() # 3. finally fix the records file self._fix_records(new_files) diff --git a/src/virtualenv/seed/via_app_data/pip_install/copy.py b/src/virtualenv/seed/embed/via_app_data/pip_install/copy.py similarity index 100% rename from src/virtualenv/seed/via_app_data/pip_install/copy.py rename to src/virtualenv/seed/embed/via_app_data/pip_install/copy.py diff --git a/src/virtualenv/seed/via_app_data/pip_install/symlink.py b/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py similarity index 100% rename from src/virtualenv/seed/via_app_data/pip_install/symlink.py rename to src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py diff --git a/src/virtualenv/seed/embed/via_app_data/via_app_data.py b/src/virtualenv/seed/embed/via_app_data/via_app_data.py new file mode 100644 index 000000000..779ee1846 --- /dev/null +++ b/src/virtualenv/seed/embed/via_app_data/via_app_data.py @@ -0,0 +1,127 @@ +"""Bootstrap""" +from __future__ import absolute_import, unicode_literals + +import logging +from contextlib import contextmanager +from subprocess import CalledProcessError +from threading import Lock, Thread + +import six + +from virtualenv.info import fs_supports_symlink +from virtualenv.seed.embed.base_embed import BaseEmbed +from virtualenv.seed.wheels import get_wheel +from virtualenv.util.path import Path + +from .pip_install.copy import CopyPipInstall +from .pip_install.symlink import SymlinkPipInstall + + +class FromAppData(BaseEmbed): + def __init__(self, options): + super(FromAppData, self).__init__(options) + self.symlinks = options.symlink_app_data + + @classmethod + def add_parser_arguments(cls, parser, interpreter, app_data): + super(FromAppData, cls).add_parser_arguments(parser, interpreter, app_data) + can_symlink = app_data.transient is False and fs_supports_symlink() + parser.add_argument( + "--symlink-app-data", + dest="symlink_app_data", + action="store_true" if can_symlink else "store_false", + help="{} symlink the python packages from the app-data folder (requires seed pip>=19.3)".format( + "" if can_symlink else "not supported - ", + ), + default=False, + ) + + def run(self, creator): + if not self.enabled: + return + with self._get_seed_wheels(creator) as name_to_whl: + pip_version = name_to_whl["pip"].version_tuple if "pip" in name_to_whl else None + installer_class = self.installer_class(pip_version) + + def _install(name, wheel): + logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) + key = Path(installer_class.__name__) / wheel.path.stem + wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key) + installer = installer_class(wheel.path, creator, wheel_img) + if not installer.has_image(): + installer.build_image() + installer.install(creator.interpreter.version_info) + + threads = list(Thread(target=_install, args=(n, w)) for n, w in name_to_whl.items()) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + @contextmanager + def _get_seed_wheels(self, creator): + name_to_whl, lock, fail = {}, Lock(), {} + + def _get(distribution, version): + for_py_version = creator.interpreter.version_release_str + failure, result = None, None + # fallback to download in case the exact version is not available + for download in [True] if self.download else [False, True]: + failure = None + try: + result = get_wheel( + distribution=distribution, + version=version, + for_py_version=for_py_version, + search_dirs=self.extra_search_dir, + download=download, + app_data=self.app_data, + do_periodic_update=self.periodic_update, + ) + if result is not None: + break + except Exception as exception: # noqa + logging.exception("fail") + failure = exception + if failure: + if isinstance(failure, CalledProcessError): + msg = "failed to download {}".format(distribution) + if version is not None: + msg += " version {}".format(version) + msg += ", pip download exit code {}".format(failure.returncode) + output = failure.output if six.PY2 else (failure.output + failure.stderr) + if output: + msg += "\n" + msg += output + else: + msg = repr(failure) + logging.error(msg) + with lock: + fail[distribution] = version + else: + with lock: + name_to_whl[distribution] = result + + threads = list( + Thread(target=_get, args=(distribution, version)) + for distribution, version in self.distribution_to_versions().items() + ) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + if fail: + raise RuntimeError("seed failed due to failing to download wheels {}".format(", ".join(fail.keys()))) + yield name_to_whl + + def installer_class(self, pip_version_tuple): + if self.symlinks and pip_version_tuple: + # symlink support requires pip 19.3+ + if pip_version_tuple >= (19, 3): + return SymlinkPipInstall + return CopyPipInstall + + def __unicode__(self): + base = super(FromAppData, self).__unicode__() + msg = ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.app_data) + return base[:-1] + msg + base[-1] diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py deleted file mode 100644 index 91b630dd4..000000000 --- a/src/virtualenv/seed/embed/wheels/acquire.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Bootstrap""" -from __future__ import absolute_import, unicode_literals - -import logging -import os -import sys -from collections import defaultdict -from contextlib import contextmanager -from copy import copy -from shutil import copy2 -from zipfile import ZipFile - -from virtualenv.info import IS_ZIPAPP -from virtualenv.util.path import Path -from virtualenv.util.six import ensure_str, ensure_text -from virtualenv.util.subprocess import Popen, subprocess -from virtualenv.util.zipapp import ensure_file_on_disk - -from . import BUNDLE_SUPPORT, MAX - -BUNDLE_FOLDER = Path(os.path.abspath(__file__)).parent - - -class WheelDownloadFail(ValueError): - def __init__(self, packages, for_py_version, exit_code, out, err): - self.packages = packages - self.for_py_version = for_py_version - self.exit_code = exit_code - self.out = out.strip() - self.err = err.strip() - - -def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_data, download): - # not all wheels are compatible with all python versions, so we need to py version qualify it - processed = copy(packages) - # 1. acquire from bundle - acquire_from_bundle(processed, for_py_version, wheel_cache_dir) - # 2. acquire from extra search dir - acquire_from_dir(processed, for_py_version, wheel_cache_dir, extra_search_dir) - # 3. download from the internet - if download and processed: - download_wheel(processed, for_py_version, wheel_cache_dir, app_data) - - # in the end just get the wheels - wheels = _get_wheels(wheel_cache_dir, packages) - return {p: next(iter(ver_to_files))[1] for p, ver_to_files in wheels.items()} - - -def acquire_from_bundle(packages, for_py_version, to_folder): - for pkg, version in list(packages.items()): - bundle = get_bundled_wheel(pkg, for_py_version) - if bundle is not None: - pkg_version = bundle.stem.split("-")[1] - exact_version_match = version == pkg_version - if exact_version_match: - del packages[pkg] - if version is None or exact_version_match: - bundled_wheel_file = to_folder / bundle.name - if not bundled_wheel_file.exists(): - logging.debug("get bundled wheel %s", bundle) - if IS_ZIPAPP: - from virtualenv.util.zipapp import extract - - extract(bundle, bundled_wheel_file) - else: - copy2(str(bundle), str(bundled_wheel_file)) - - -def get_bundled_wheel(package, version_release): - return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package) - - -def acquire_from_dir(packages, for_py_version, to_folder, extra_search_dir): - if not packages: - return - for search_dir in extra_search_dir: - wheels = _get_wheels(search_dir, packages) - for pkg, ver_wheels in wheels.items(): - stop = False - for _, filename in ver_wheels: - dest = to_folder / filename.name - if not dest.exists(): - if wheel_support_py(filename, for_py_version): - logging.debug("get extra search dir wheel %s", filename) - copy2(str(filename), str(dest)) - stop = True - else: - stop = True - if stop and packages[pkg] is not None: - del packages[pkg] - break - - -def wheel_support_py(filename, py_version): - name = "{}.dist-info/METADATA".format("-".join(filename.stem.split("-")[0:2])) - with ZipFile(ensure_text(str(filename)), "r") as zip_file: - metadata = zip_file.read(name).decode("utf-8") - marker = "Requires-Python:" - requires = next((i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)), None) - if requires is None: # if it does not specify a python requires the assumption is compatible - return True - py_version_int = tuple(int(i) for i in py_version.split(".")) - for require in (i.strip() for i in requires.split(",")): - # https://www.python.org/dev/peps/pep-0345/#version-specifiers - for operator, check in [ - ("!=", lambda v: py_version_int != v), - ("==", lambda v: py_version_int == v), - ("<=", lambda v: py_version_int <= v), - (">=", lambda v: py_version_int >= v), - ("<", lambda v: py_version_int < v), - (">", lambda v: py_version_int > v), - ]: - if require.startswith(operator): - ver_str = require[len(operator) :].strip() - version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2] - if not check(version): - return False - break - return True - - -def _get_wheels(from_folder, packages): - wheels = defaultdict(list) - for filename in from_folder.iterdir(): - if filename.suffix == ".whl": - data = filename.stem.split("-") - if len(data) >= 2: - pkg, version = data[0:2] - if pkg in packages: - pkg_version = packages[pkg] - if pkg_version is None or pkg_version == version: - wheels[pkg].append((version, filename)) - for versions in wheels.values(): - versions.sort( - key=lambda a: tuple(int(i) if i.isdigit() else i for i in a[0].split(".")), reverse=True, - ) - return wheels - - -def download_wheel(packages, for_py_version, to_folder, app_data): - to_download = list(p if v is None else "{}=={}".format(p, v) for p, v in packages.items()) - logging.debug("download wheels %s", to_download) - cmd = [ - sys.executable, - "-m", - "pip", - "download", - "--disable-pip-version-check", - "--only-binary=:all:", - "--no-deps", - "--python-version", - for_py_version, - "-d", - str(to_folder), - ] - cmd.extend(to_download) - # pip has no interface in python - must be a new sub-process - - with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), app_data) as env: - process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) - out, err = process.communicate() - if process.returncode != 0: - raise WheelDownloadFail(packages, for_py_version, process.returncode, out, err) - - -@contextmanager -def pip_wheel_env_run(version, app_data): - env = os.environ.copy() - env.update( - { - ensure_str(k): str(v) # python 2 requires these to be string only (non-unicode) - for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items() - }, - ) - with ensure_file_on_disk(get_bundled_wheel("pip", version), app_data) as pip_wheel_path: - # put the bundled wheel onto the path, and use it to do the bootstrap operation - env[str("PYTHONPATH")] = str(pip_wheel_path) - yield env diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py deleted file mode 100644 index de3757d7c..000000000 --- a/src/virtualenv/seed/via_app_data/via_app_data.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Bootstrap""" -from __future__ import absolute_import, unicode_literals - -import logging -from contextlib import contextmanager -from functools import partial -from threading import Lock, Thread - -from virtualenv.info import fs_supports_symlink -from virtualenv.seed.embed.base_embed import BaseEmbed -from virtualenv.seed.embed.wheels.acquire import WheelDownloadFail, get_wheels -from virtualenv.util.path import safe_delete - -from .pip_install.copy import CopyPipInstall -from .pip_install.symlink import SymlinkPipInstall - - -class FromAppData(BaseEmbed): - def __init__(self, options): - super(FromAppData, self).__init__(options) - self.symlinks = options.symlink_app_data - self.base_cache = self.app_data / "seed-app-data" / "v1.0.1" - - @classmethod - def add_parser_arguments(cls, parser, interpreter, app_data): - super(FromAppData, cls).add_parser_arguments(parser, interpreter, app_data) - can_symlink = app_data.transient is False and fs_supports_symlink() - parser.add_argument( - "--symlink-app-data", - dest="symlink_app_data", - action="store_true" if can_symlink else "store_false", - help="{} symlink the python packages from the app-data folder (requires seed pip>=19.3)".format( - "" if can_symlink else "not supported - ", - ), - default=False, - ) - - def run(self, creator): - if not self.enabled: - return - base_cache = self.base_cache / creator.interpreter.version_release_str - with self._get_seed_wheels(creator, base_cache) as name_to_whl: - pip_version = name_to_whl["pip"].stem.split("-")[1] if "pip" in name_to_whl else None - installer_class = self.installer_class(pip_version) - - def _install(name, wheel): - logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) - image_folder = base_cache.path / "image" / installer_class.__name__ / wheel.stem - installer = installer_class(wheel, creator, image_folder) - if not installer.has_image(): - installer.build_image() - installer.install(creator.interpreter.version_info) - - threads = list(Thread(target=_install, args=(n, w)) for n, w in name_to_whl.items()) - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - @contextmanager - def _get_seed_wheels(self, creator, base_cache): - with base_cache.lock_for_key("wheels"): - wheels_to = base_cache.path / "wheels" - if wheels_to.exists(): - safe_delete(wheels_to) - wheels_to.mkdir(parents=True, exist_ok=True) - name_to_whl, lock, fail = {}, Lock(), {} - - def _get(package, version): - wheel_loader = partial( - get_wheels, - creator.interpreter.version_release_str, - wheels_to, - self.extra_search_dir, - {package: version}, - self.app_data, - ) - failure, result = None, None - # fallback to download in case the exact version is not available - for download in [True] if self.download else [False, True]: - failure = None - try: - result = wheel_loader(download) - if result: - break - except Exception as exception: - failure = exception - if failure: - if isinstance(failure, WheelDownloadFail): - msg = "failed to download {}".format(package) - if version is not None: - msg += " version {}".format(version) - msg += ", pip download exit code {}".format(failure.exit_code) - output = failure.out + failure.err - if output: - msg += "\n" - msg += output - else: - msg = repr(failure) - logging.error(msg) - with lock: - fail[package] = version - else: - with lock: - name_to_whl.update(result) - - package_versions = self.package_version() - threads = list(Thread(target=_get, args=(pkg, v)) for pkg, v in package_versions.items()) - for thread in threads: - thread.start() - for thread in threads: - thread.join() - if fail: - raise RuntimeError("seed failed due to failing to download wheels {}".format(", ".join(fail.keys()))) - yield name_to_whl - - def installer_class(self, pip_version): - if self.symlinks and pip_version: - # symlink support requires pip 19.3+ - pip_version_int = tuple(int(i) for i in pip_version.split(".")[0:2]) - if pip_version_int >= (19, 3): - return SymlinkPipInstall - return CopyPipInstall - - def __unicode__(self): - base = super(FromAppData, self).__unicode__() - msg = ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.base_cache.path) - return base[:-1] + msg + base[-1] diff --git a/src/virtualenv/seed/wheels/__init__.py b/src/virtualenv/seed/wheels/__init__.py new file mode 100644 index 000000000..dbffe2e43 --- /dev/null +++ b/src/virtualenv/seed/wheels/__init__.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, unicode_literals + +from .acquire import get_wheel, pip_wheel_env_run +from .util import Version, Wheel + +__all__ = ( + "get_wheel", + "pip_wheel_env_run", + "Version", + "Wheel", +) diff --git a/src/virtualenv/seed/wheels/acquire.py b/src/virtualenv/seed/wheels/acquire.py new file mode 100644 index 000000000..8c88725f9 --- /dev/null +++ b/src/virtualenv/seed/wheels/acquire.py @@ -0,0 +1,114 @@ +"""Bootstrap""" +from __future__ import absolute_import, unicode_literals + +import logging +import os +import sys +from operator import eq, lt + +import six + +from virtualenv.util.path import Path +from virtualenv.util.six import ensure_str +from virtualenv.util.subprocess import Popen, subprocess + +from .bundle import from_bundle +from .util import Version, Wheel, discover_wheels + + +def get_wheel(distribution, version, for_py_version, search_dirs, download, app_data, do_periodic_update): + """ + Get a wheel with the given distribution-version-for_py_version trio, by using the extra search dir + download + """ + # not all wheels are compatible with all python versions, so we need to py version qualify it + # 1. acquire from bundle + wheel = from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update) + + # 2. download from the internet + if version not in Version.non_version and download: + wheel = download_wheel( + distribution=distribution, + version_spec=Version.as_version_spec(version), + for_py_version=for_py_version, + search_dirs=search_dirs, + app_data=app_data, + to_folder=app_data.house, + ) + return wheel + + +def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder): + to_download = "{}{}".format(distribution, version_spec or "") + logging.debug("download wheel %s", to_download) + cmd = [ + sys.executable, + "-m", + "pip", + "download", + "--disable-pip-version-check", + "--only-binary=:all:", + "--no-deps", + "--python-version", + for_py_version, + "-d", + str(to_folder), + to_download, + ] + # pip has no interface in python - must be a new sub-process + env = pip_wheel_env_run(search_dirs, app_data) + process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + out, err = process.communicate() + if process.returncode != 0: + kwargs = {"output": out} + if six.PY2: + kwargs["output"] += err + else: + kwargs["stderr"] = err + raise subprocess.CalledProcessError(process.returncode, cmd, **kwargs) + for line in out.splitlines(): + line = line.lstrip() + for marker in ("Saved ", "File was already downloaded "): + if line.startswith(marker): + return Wheel(Path(line[len(marker) :]).absolute()) + # if for some reason the output does not match fallback to latest version with that spec + return find_compatible_in_house(distribution, version_spec, for_py_version, to_folder) + + +def find_compatible_in_house(distribution, version_spec, for_py_version, in_folder): + wheels = discover_wheels(in_folder, distribution, None, for_py_version) + start, end = 0, len(wheels) + if version_spec is not None: + if version_spec.startswith("<"): + from_pos, op = 1, lt + elif version_spec.startswith("=="): + from_pos, op = 2, eq + else: + raise ValueError(version_spec) + version = Wheel.as_version_tuple(version_spec[from_pos:]) + start = next((at for at, w in enumerate(wheels) if op(w.version_tuple, version)), len(wheels)) + + return None if start == end else wheels[start] + + +def pip_wheel_env_run(search_dirs, app_data): + for_py_version = "{}.{}".format(*sys.version_info[0:2]) + env = os.environ.copy() + env.update( + { + ensure_str(k): str(v) # python 2 requires these to be string only (non-unicode) + for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items() + }, + ) + wheel = get_wheel( + distribution="pip", + version=None, + for_py_version=for_py_version, + search_dirs=search_dirs, + download=False, + app_data=app_data, + do_periodic_update=False, + ) + if wheel is None: + raise RuntimeError("could not find the embedded pip") + env[str("PYTHONPATH")] = str(wheel.path) + return env diff --git a/src/virtualenv/seed/wheels/bundle.py b/src/virtualenv/seed/wheels/bundle.py new file mode 100644 index 000000000..6ac15f9ec --- /dev/null +++ b/src/virtualenv/seed/wheels/bundle.py @@ -0,0 +1,51 @@ +from __future__ import absolute_import, unicode_literals + +from virtualenv.app_data import AppDataDiskFolder, TempAppData + +from ..wheels.embed import get_embed_wheel +from .periodic_update import periodic_update +from .util import Version, Wheel, discover_wheels + + +def from_bundle(distribution, version, for_py_version, search_dirs, app_data, do_periodic_update): + """ + Load the bundled wheel to a cache directory. + """ + of_version = Version.of_version(version) + wheel = load_embed_wheel(app_data, distribution, for_py_version, of_version) + + if version != Version.embed: + # 2. check if we have upgraded embed + if isinstance(app_data, AppDataDiskFolder) and not isinstance(app_data, TempAppData): + wheel = periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update) + + # 3. acquire from extra search dir + found_wheel = from_dir(distribution, of_version, for_py_version, search_dirs) + if found_wheel is not None: + if wheel is None: + wheel = found_wheel + elif found_wheel.version_tuple > wheel.version_tuple: + wheel = found_wheel + return wheel + + +def load_embed_wheel(app_data, distribution, for_py_version, version): + wheel = get_embed_wheel(distribution, for_py_version) + if wheel is not None: + version_match = version == wheel.version + if version is None or version_match: + with app_data.ensure_extracted(wheel.path, lambda: app_data.house) as wheel_path: + wheel = Wheel(wheel_path) + else: # if version does not match ignore + wheel = None + return wheel + + +def from_dir(distribution, version, for_py_version, directories): + """ + Load a compatible wheel from a given folder. + """ + for folder in directories: + for wheel in discover_wheels(folder, distribution, version, for_py_version): + return wheel + return None diff --git a/src/virtualenv/seed/embed/wheels/__init__.py b/src/virtualenv/seed/wheels/embed/__init__.py similarity index 68% rename from src/virtualenv/seed/embed/wheels/__init__.py rename to src/virtualenv/seed/wheels/embed/__init__.py index 28fd0c476..17860e0eb 100644 --- a/src/virtualenv/seed/embed/wheels/__init__.py +++ b/src/virtualenv/seed/wheels/embed/__init__.py @@ -1,6 +1,15 @@ from __future__ import absolute_import, unicode_literals +from virtualenv.seed.wheels.util import Wheel +from virtualenv.util.path import Path + +BUNDLE_FOLDER = Path(__file__).absolute().parent BUNDLE_SUPPORT = { + "3.10": { + "pip": "pip-20.1.1-py2.py3-none-any.whl", + "setuptools": "setuptools-47.1.1-py3-none-any.whl", + "wheel": "wheel-0.34.2-py2.py3-none-any.whl", + }, "3.9": { "pip": "pip-20.1.1-py2.py3-none-any.whl", "setuptools": "setuptools-47.1.1-py3-none-any.whl", @@ -37,4 +46,17 @@ "wheel": "wheel-0.34.2-py2.py3-none-any.whl", }, } -MAX = "3.9" +MAX = "3.10" + + +def get_embed_wheel(distribution, for_py_version): + path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX]).get(distribution) + return Wheel.from_path(path) + + +__all__ = ( + "get_embed_wheel", + "BUNDLE_SUPPORT", + "MAX", + "BUNDLE_FOLDER", +) diff --git a/src/virtualenv/seed/embed/wheels/pip-19.1.1-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-19.1.1-py2.py3-none-any.whl similarity index 100% rename from src/virtualenv/seed/embed/wheels/pip-19.1.1-py2.py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/pip-19.1.1-py2.py3-none-any.whl diff --git a/src/virtualenv/seed/embed/wheels/pip-20.1.1-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-20.1.1-py2.py3-none-any.whl similarity index 100% rename from src/virtualenv/seed/embed/wheels/pip-20.1.1-py2.py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/pip-20.1.1-py2.py3-none-any.whl diff --git a/src/virtualenv/seed/embed/wheels/setuptools-43.0.0-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-43.0.0-py2.py3-none-any.whl similarity index 100% rename from src/virtualenv/seed/embed/wheels/setuptools-43.0.0-py2.py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/setuptools-43.0.0-py2.py3-none-any.whl diff --git a/src/virtualenv/seed/embed/wheels/setuptools-44.1.1-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-44.1.1-py2.py3-none-any.whl similarity index 100% rename from src/virtualenv/seed/embed/wheels/setuptools-44.1.1-py2.py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/setuptools-44.1.1-py2.py3-none-any.whl diff --git a/src/virtualenv/seed/embed/wheels/setuptools-47.1.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-47.1.1-py3-none-any.whl similarity index 100% rename from src/virtualenv/seed/embed/wheels/setuptools-47.1.1-py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/setuptools-47.1.1-py3-none-any.whl diff --git a/src/virtualenv/seed/embed/wheels/wheel-0.33.6-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.33.6-py2.py3-none-any.whl similarity index 100% rename from src/virtualenv/seed/embed/wheels/wheel-0.33.6-py2.py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/wheel-0.33.6-py2.py3-none-any.whl diff --git a/src/virtualenv/seed/embed/wheels/wheel-0.34.2-py2.py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.34.2-py2.py3-none-any.whl similarity index 100% rename from src/virtualenv/seed/embed/wheels/wheel-0.34.2-py2.py3-none-any.whl rename to src/virtualenv/seed/wheels/embed/wheel-0.34.2-py2.py3-none-any.whl diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py new file mode 100644 index 000000000..25270ad6d --- /dev/null +++ b/src/virtualenv/seed/wheels/periodic_update.py @@ -0,0 +1,311 @@ +""" +Periodically update bundled versions. +""" + +from __future__ import absolute_import, unicode_literals + +import json +import logging +import os +import subprocess +import sys +from datetime import datetime, timedelta +from itertools import groupby +from shutil import copy2 +from threading import Thread + +from six.moves.urllib.request import urlopen + +from virtualenv.app_data import AppDataDiskFolder +from virtualenv.info import PY2 +from virtualenv.util.path import Path +from virtualenv.util.subprocess import DETACHED_PROCESS, Popen + +from ..wheels.embed import BUNDLE_SUPPORT +from ..wheels.util import Wheel + +if PY2: + # on Python 2 datetime.strptime throws the error below if the import did not trigger on main thread + # Failed to import _strptime because the import lock is held by + try: + import _strptime # noqa + except ImportError: # pragma: no cov + pass # pragma: no cov + + +def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update): + if do_periodic_update: + handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data) + + now = datetime.now() + + u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) + u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False + for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]): + version = next(group) # use only latest patch version per minor, earlier assumed to be buggy + if wheel is not None and Path(version.filename).name == wheel.name: + break + if u_log.periodic is False or (u_log_older_than_hour and version.use(now)): + updated_wheel = Wheel(app_data.house / version.filename) + logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) + wheel = updated_wheel + break + + return wheel + + +def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data): + embed_update_log = app_data.embed_update_log(distribution, for_py_version) + u_log = UpdateLog.from_dict(embed_update_log.read()) + if u_log.needs_update: + u_log.periodic = True + u_log.started = datetime.now() + embed_update_log.write(u_log.to_dict()) + trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True) + + +DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +def dump_datetime(value): + return None if value is None else value.strftime(DATETIME_FMT) + + +def load_datetime(value): + return None if value is None else datetime.strptime(value, DATETIME_FMT) + + +class NewVersion(object): + def __init__(self, filename, found_date, release_date): + self.filename = filename + self.found_date = found_date + self.release_date = release_date + + @classmethod + def from_dict(cls, dictionary): + return cls( + filename=dictionary["filename"], + found_date=load_datetime(dictionary["found_date"]), + release_date=load_datetime(dictionary["release_date"]), + ) + + def to_dict(self): + return { + "filename": self.filename, + "release_date": dump_datetime(self.release_date), + "found_date": dump_datetime(self.found_date), + } + + def use(self, now): + compare_from = self.release_date or self.found_date + return now - compare_from >= timedelta(days=28) + + def __repr__(self): + return "{}(filename={}), found_date={}, release_date={})".format( + self.__class__.__name__, self.filename, self.found_date, self.release_date, + ) + + def __eq__(self, other): + return type(self) == type(other) and all( + getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"] + ) + + def __ne__(self, other): + return not (self == other) + + @property + def wheel(self): + return Wheel(Path(self.filename)) + + +class UpdateLog(object): + def __init__(self, started, completed, versions, periodic): + self.started = started + self.completed = completed + self.versions = versions + self.periodic = periodic + + @classmethod + def from_dict(cls, dictionary): + if dictionary is None: + dictionary = {} + return cls( + load_datetime(dictionary.get("started")), + load_datetime(dictionary.get("completed")), + [NewVersion.from_dict(v) for v in dictionary.get("versions", [])], + dictionary.get("periodic"), + ) + + @classmethod + def from_app_data(cls, app_data, distribution, for_py_version): + raw_json = app_data.embed_update_log(distribution, for_py_version).read() + return cls.from_dict(raw_json) + + def to_dict(self): + return { + "started": dump_datetime(self.started), + "completed": dump_datetime(self.completed), + "periodic": self.periodic, + "versions": [r.to_dict() for r in self.versions], + } + + @property + def needs_update(self): + now = datetime.now() + if self.completed is None: # never completed + return self._check_start(now) + else: + if now - self.completed <= timedelta(days=14): + return False + return self._check_start(now) + + def _check_start(self, now): + return self.started is None or now - self.started > timedelta(hours=1) + + +def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic): + wheel_path = None if wheel is None else str(wheel.path) + cmd = [ + sys.executable, + "-c", + "from virtualenv.seed.wheels.periodic_update import do_update;" + "do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".format( + distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic, + ), + ] + debug = os.environ.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1") + pipe = None if debug else subprocess.PIPE + kwargs = {"stdout": pipe, "stderr": pipe} + if not debug and sys.platform == "win32": + kwargs["creationflags"] = DETACHED_PROCESS + process = Popen(cmd, **kwargs) + logging.info( + "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", + distribution, + "" if wheel is None else "=={}".format(wheel.version), + for_py_version, + process.pid, + ) + if debug: + process.communicate() # on purpose not called to make it a background process + + +def do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): + + from virtualenv.seed.wheels import acquire + + wheel_filename = None if embed_filename is None else Path(embed_filename) + app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data + search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs] + wheelhouse = app_data.house + embed_update_log = app_data.embed_update_log(distribution, for_py_version) + u_log = UpdateLog.from_dict(embed_update_log.read()) + + now = datetime.now() + + if wheel_filename is not None: + dest = wheelhouse / wheel_filename.name + if not dest.exists(): + copy2(str(wheel_filename), str(wheelhouse)) + + last, versions = None, [] + while last is None or not last.use(now): + download_time = datetime.now() + dest = acquire.download_wheel( + distribution=distribution, + version_spec=None if last is None else "<{}".format(Wheel(Path(last.filename)).version), + for_py_version=for_py_version, + search_dirs=search_dirs, + app_data=app_data, + to_folder=wheelhouse, + ) + if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name): + break + release_date = _get_release_date(dest.path) + last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time) + logging.info("detected %s in %s", last, datetime.now() - download_time) + versions.append(last) + u_log.periodic = periodic + if not u_log.periodic: + u_log.started = now + u_log.versions = versions + u_log.versions + u_log.completed = datetime.now() + embed_update_log.write(u_log.to_dict()) + return versions + + +def _get_release_date(dest): + wheel = Wheel(dest) + # the most accurate is to ask PyPi - e.g. https://pypi.org/pypi/pip/json, + # see https://warehouse.pypa.io/api-reference/json/ for more details + try: + with urlopen("https://pypi.org/pypi/{}/json".format(wheel.distribution)) as file_handler: + content = json.load(file_handler) + return datetime.strptime(content["releases"][wheel.version][0]["upload_time"], "%Y-%m-%dT%H:%M:%S") + except Exception: # noqa + return None + + +def manual_upgrade(app_data): + threads = [] + + for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items(): + # load extra search dir for the given for_py + for distribution in distribution_to_package.keys(): + thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version)) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + +def _run_manual_upgrade(app_data, distribution, for_py_version): + start = datetime.now() + from .bundle import from_bundle + + current = from_bundle( + distribution=distribution, + version=None, + for_py_version=for_py_version, + search_dirs=[], + app_data=app_data, + do_periodic_update=False, + ) + logging.warning( + "upgrade %s for python %s with current %s", + distribution, + for_py_version, + "" if current is None else current.name, + ) + versions = do_update( + distribution=distribution, + for_py_version=for_py_version, + embed_filename=current.path, + app_data=app_data, + search_dirs=[], + periodic=False, + ) + msg = "upgraded %s for python %s in %s {}".format( + "new entries found:\n%s" if versions else "no new versions found", + ) + args = [ + distribution, + for_py_version, + datetime.now() - start, + ] + if versions: + args.append("\n".join("\t{}".format(v) for v in versions)) + logging.warning(msg, *args) + + +__all__ = ( + "periodic_update", + "do_update", + "manual_upgrade", + "NewVersion", + "UpdateLog", + "load_datetime", + "dump_datetime", + "trigger_update", +) diff --git a/src/virtualenv/seed/wheels/util.py b/src/virtualenv/seed/wheels/util.py new file mode 100644 index 000000000..1240eb2d2 --- /dev/null +++ b/src/virtualenv/seed/wheels/util.py @@ -0,0 +1,116 @@ +from __future__ import absolute_import, unicode_literals + +from operator import attrgetter +from zipfile import ZipFile + +from virtualenv.util.six import ensure_text + + +class Wheel(object): + def __init__(self, path): + # https://www.python.org/dev/peps/pep-0427/#file-name-convention + # The wheel filename is {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + self.path = path + self._parts = path.stem.split("-") + + @classmethod + def from_path(cls, path): + if path is not None and path.suffix == ".whl" and len(path.stem.split("-")) >= 5: + return cls(path) + return None + + @property + def distribution(self): + return self._parts[0] + + @property + def version(self): + return self._parts[1] + + @property + def version_tuple(self): + return self.as_version_tuple(self.version) + + @staticmethod + def as_version_tuple(version): + result = [] + for part in version.split(".")[0:3]: + try: + result.append(int(part)) + except ValueError: + break + if not result: + raise ValueError(version) + return tuple(result) + + @property + def name(self): + return self.path.name + + def support_py(self, py_version): + name = "{}.dist-info/METADATA".format("-".join(self.path.stem.split("-")[0:2])) + with ZipFile(ensure_text(str(self.path)), "r") as zip_file: + metadata = zip_file.read(name).decode("utf-8") + marker = "Requires-Python:" + requires = next((i[len(marker) :] for i in metadata.splitlines() if i.startswith(marker)), None) + if requires is None: # if it does not specify a python requires the assumption is compatible + return True + py_version_int = tuple(int(i) for i in py_version.split(".")) + for require in (i.strip() for i in requires.split(",")): + # https://www.python.org/dev/peps/pep-0345/#version-specifiers + for operator, check in [ + ("!=", lambda v: py_version_int != v), + ("==", lambda v: py_version_int == v), + ("<=", lambda v: py_version_int <= v), + (">=", lambda v: py_version_int >= v), + ("<", lambda v: py_version_int < v), + (">", lambda v: py_version_int > v), + ]: + if require.startswith(operator): + ver_str = require[len(operator) :].strip() + version = tuple((int(i) if i != "*" else None) for i in ver_str.split("."))[0:2] + if not check(version): + return False + break + return True + + def __repr__(self): + return "{}({})".format(self.__class__.__name__, self.path) + + def __str__(self): + return str(self.path) + + +def discover_wheels(from_folder, distribution, version, for_py_version): + wheels = [] + for filename in from_folder.iterdir(): + wheel = Wheel.from_path(filename) + if wheel and wheel.distribution == distribution: + if version is None or wheel.version == version: + if wheel.support_py(for_py_version): + wheels.append(wheel) + return sorted(wheels, key=attrgetter("version_tuple", "distribution"), reverse=True) + + +class Version: + #: the version bundled with virtualenv + bundle = "bundle" + embed = "embed" + #: custom version handlers + non_version = ( + bundle, + embed, + ) + + @staticmethod + def of_version(value): + return None if value in Version.non_version else value + + @staticmethod + def as_pip_req(distribution, version): + return "{}{}".format(distribution, Version.as_version_spec(version)) + + @staticmethod + def as_version_spec(version): + of_version = Version.of_version(version) + return "" if of_version is None else "=={}".format(of_version) diff --git a/src/virtualenv/error.py b/src/virtualenv/util/error.py similarity index 100% rename from src/virtualenv/error.py rename to src/virtualenv/util/error.py diff --git a/src/virtualenv/util/lock.py b/src/virtualenv/util/lock.py index 0c5e72fc7..eb7a78f95 100644 --- a/src/virtualenv/util/lock.py +++ b/src/virtualenv/util/lock.py @@ -74,7 +74,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self._release(self._lock) - def _lock_file(self, lock): + def _lock_file(self, lock, no_block=False): # multiple processes might be trying to get a first lock... so we cannot check if this directory exist without # a lock, but that lock might then become expensive, and it's not clear where that lock should live. # Instead here we just ignore if we fail to create the directory. @@ -85,6 +85,8 @@ def _lock_file(self, lock): try: lock.acquire(0.0001) except Timeout: + if no_block: + raise logging.debug("lock file %s present, will block until released", lock.lock_file) lock.release() # release the acquire try from above lock.acquire() @@ -94,13 +96,19 @@ def _release(lock): lock.release() @contextmanager - def lock_for_key(self, name): + def lock_for_key(self, name, no_block=False): lock = self._create_lock(name) try: try: - self._lock_file(lock) + self._lock_file(lock, no_block) yield finally: self._release(lock) finally: self._del_lock(lock) + + +__all__ = ( + "Timeout", + "ReentrantFileLock", +) diff --git a/src/virtualenv/util/path/_pathlib/via_os_path.py b/src/virtualenv/util/path/_pathlib/via_os_path.py index 3afbe3504..d11aeaaab 100644 --- a/src/virtualenv/util/path/_pathlib/via_os_path.py +++ b/src/virtualenv/util/path/_pathlib/via_os_path.py @@ -74,8 +74,11 @@ def is_dir(self): return os.path.isdir(self._path) def mkdir(self, parents=True, exist_ok=True): - if not self.exists() and exist_ok: + try: os.makedirs(self._path) + except OSError: + if not exist_ok: + raise def read_text(self, encoding="utf-8"): return self.read_bytes().decode(encoding) @@ -135,5 +138,8 @@ def stat(self): def chmod(self, mode): os.chmod(self._path, mode) + def absolute(self): + return Path(os.path.abspath(self._path)) + __all__ = ("Path",) diff --git a/src/virtualenv/util/subprocess/__init__.py b/src/virtualenv/util/subprocess/__init__.py index 6cb0a2810..22006da84 100644 --- a/src/virtualenv/util/subprocess/__init__.py +++ b/src/virtualenv/util/subprocess/__init__.py @@ -13,6 +13,9 @@ Popen = subprocess.Popen +DETACHED_PROCESS = 0x00000008 + + def run_cmd(cmd): try: process = Popen( @@ -29,4 +32,5 @@ def run_cmd(cmd): "subprocess", "Popen", "run_cmd", + "DETACHED_PROCESS", ) diff --git a/src/virtualenv/util/zipapp.py b/src/virtualenv/util/zipapp.py index 36cee5e78..85d9294f4 100644 --- a/src/virtualenv/util/zipapp.py +++ b/src/virtualenv/util/zipapp.py @@ -3,13 +3,9 @@ import logging import os import zipfile -from contextlib import contextmanager -from tempfile import TemporaryFile -from virtualenv.info import IS_WIN, IS_ZIPAPP, ROOT -from virtualenv.util.path import Path +from virtualenv.info import IS_WIN, ROOT from virtualenv.util.six import ensure_text -from virtualenv.version import __version__ def read(full_path): @@ -35,22 +31,3 @@ def _get_path_within_zip(full_path): # paths are always UNIX separators, even on Windows, though __file__ still follows platform default sub_file = sub_file.replace(os.sep, "/") return sub_file - - -@contextmanager -def ensure_file_on_disk(path, app_data): - if IS_ZIPAPP: - if app_data is None: - with TemporaryFile() as temp_file: - dest = Path(temp_file.name) - extract(path, dest) - yield Path(dest) - else: - base = app_data / "zipapp" / "extract" / __version__ - with base.lock_for_key(path.name): - dest = base.path / path.name - if not dest.exists(): - extract(path, dest) - yield dest - else: - yield path diff --git a/tasks/upgrade_wheels.py b/tasks/upgrade_wheels.py index 8de3eeaef..fe0010ab8 100644 --- a/tasks/upgrade_wheels.py +++ b/tasks/upgrade_wheels.py @@ -10,18 +10,31 @@ from collections import OrderedDict, defaultdict from pathlib import Path from tempfile import TemporaryDirectory +from textwrap import dedent from threading import Thread STRICT = "UPGRADE_ADVISORY" not in os.environ BUNDLED = ["pip", "setuptools", "wheel"] -SUPPORT = list(reversed([(2, 7)] + [(3, i) for i in range(4, 10)])) -DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "embed" / "wheels" +SUPPORT = list(reversed([(2, 7)] + [(3, i) for i in range(4, 11)])) +DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "wheels" / "embed" def download(ver, dest, package): subprocess.call( - [sys.executable, "-m", "pip", "download", "--only-binary=:all:", "--python-version", ver, "-d", dest, package], + [ + sys.executable, + "-m", + "pip", + "--disable-pip-version-check", + "download", + "--only-binary=:all:", + "--python-version", + ver, + "-d", + dest, + package, + ], ) @@ -72,12 +85,37 @@ def run(): support_table[version].append(package) support_table = {k: OrderedDict((i.split("-")[0], i) for i in v) for k, v in support_table.items()} - msg = "from __future__ import absolute_import, unicode_literals; BUNDLE_SUPPORT = {{ {} }}; MAX = {!r}".format( - ",".join( - "{!r}: {{ {} }}".format(v, ",".join("{!r}: {!r}".format(p, f) for p, f in l.items())) - for v, l in support_table.items() + msg = dedent( + """ + from __future__ import absolute_import, unicode_literals + + from virtualenv.seed.wheels.util import Wheel + from virtualenv.util.path import Path + + BUNDLE_FOLDER = Path(__file__).absolute().parent + BUNDLE_SUPPORT = {{ {0} }} + MAX = {1} + + + def get_embed_wheel(distribution, for_py_version): + path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {{}}) or BUNDLE_SUPPORT[MAX]).get(distribution) + return Wheel.from_path(path) + + + __all__ = ( + "get_embed_wheel", + "BUNDLE_SUPPORT", + "MAX", + "BUNDLE_FOLDER", + ) + + """.format( + ",".join( + "{!r}: {{ {} }}".format(v, ",".join("{!r}: {!r}".format(p, f) for p, f in l.items())) + for v, l in support_table.items() + ), + repr(next(iter(support_table.keys()))), ), - next(iter(support_table.keys())), ) dest_target = DEST / "__init__.py" dest_target.write_text(msg) diff --git a/tests/conftest.py b/tests/conftest.py index a92b893b1..97e109e93 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,11 +12,11 @@ import pytest import six +from virtualenv.app_data import AppDataDiskFolder from virtualenv.discovery.builtin import get_interpreter from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink from virtualenv.report import LOGGER -from virtualenv.run.app_data import AppData from virtualenv.util.path import Path from virtualenv.util.six import ensure_str, ensure_text @@ -145,7 +145,12 @@ def check_os_environ_stable(): if k.startswith(str("VIRTUALENV_")) or str("VIRTUAL_ENV") in k or k.startswith(str("TOX_")) } cleaned = {k: os.environ[k] for k, v in os.environ.items()} - os.environ[str("VIRTUALENV_NO_DOWNLOAD")] = str("1") + override = { + "VIRTUALENV_NO_PERIODIC_UPDATE": "1", + "VIRTUALENV_NO_DOWNLOAD": "1", + } + for key, value in override.items(): + os.environ[str(key)] = str(value) is_exception = False try: yield @@ -154,7 +159,8 @@ def check_os_environ_stable(): raise finally: try: - del os.environ[str("VIRTUALENV_NO_DOWNLOAD")] + for key in override.keys(): + del os.environ[str(key)] if is_exception is False: new = os.environ extra = {k: new[k] for k in set(new) - set(old)} @@ -291,9 +297,9 @@ def current_fastest(current_creators): @pytest.fixture(scope="session") def session_app_data(tmp_path_factory): - app_data = AppData(folder=str(tmp_path_factory.mktemp("session-app-data"))) - with change_env_var(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(app_data.folder.path)): - yield app_data.folder + app_data = AppDataDiskFolder(folder=str(tmp_path_factory.mktemp("session-app-data"))) + with change_env_var(str("VIRTUALENV_OVERRIDE_APP_DATA"), str(app_data.lock.path)): + yield app_data @contextmanager @@ -331,3 +337,8 @@ def cross_python(is_inside_ci, session_app_data): raise RuntimeError(msg) pytest.skip(msg=msg) yield interpreter + + +@pytest.fixture(scope="session") +def for_py_version(): + return "{}.{}".format(*sys.version_info[0:2]) diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py index 605496ef7..e5198849d 100644 --- a/tests/integration/test_zipapp.py +++ b/tests/integration/test_zipapp.py @@ -38,6 +38,7 @@ def zipapp_build_env(tmp_path_factory): "", str(create_env_path), "--no-download", + "--no-periodic-update", ], ) exe = str(session.creator.exe) @@ -70,7 +71,7 @@ def zipapp(zipapp_build_env, tmp_path_factory): @pytest.fixture(scope="session") def zipapp_test_env(tmp_path_factory): base_path = tmp_path_factory.mktemp("zipapp-test") - session = cli_run(["-v", "--activators", "", "--without-pip", str(base_path / "env")]) + session = cli_run(["-v", "--activators", "", "--without-pip", str(base_path / "env"), "--no-periodic-update"]) yield session.creator.exe shutil.rmtree(str(base_path)) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 826ea3d34..98c3fda55 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -74,7 +74,8 @@ def __call__(self, monkeypatch, tmp_path): _raw, _ = process.communicate() raw = _raw.decode("utf-8") except subprocess.CalledProcessError as exception: - assert not exception.returncode, ensure_text(exception.output) + output = ensure_text((exception.output + exception.stderr) if six.PY3 else exception.output) + assert not exception.returncode, output return out = re.sub(r"pydev debugger: process \d+ is connecting\n\n", "", raw, re.M).strip().splitlines() @@ -208,7 +209,7 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session", params=[True, False], ids=["with_prompt", "no_prompt"]) def activation_python(request, tmp_path_factory, special_char_name, current_fastest): dest = os.path.join(ensure_text(str(tmp_path_factory.mktemp("activation-tester-env"))), special_char_name) - cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv"] + cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv", "--no-periodic-update"] if request.param: cmd += ["--prompt", special_char_name] session = cli_run(cmd) diff --git a/tests/unit/activation/test_activate_this.py b/tests/unit/activation/test_activate_this.py index 9446a4c48..53f4b3f53 100644 --- a/tests/unit/activation/test_activate_this.py +++ b/tests/unit/activation/test_activate_this.py @@ -10,7 +10,7 @@ def test_python_activator_cross(session_app_data, cross_python, special_name_dir "-p", str(cross_python.executable), "--app-data", - str(session_app_data.path), + str(session_app_data.lock.path), "--without-pip", "--activators", "", diff --git a/tests/unit/activation/test_xonsh.py b/tests/unit/activation/test_xonsh.py index 3945d7c8b..60df8a46b 100644 --- a/tests/unit/activation/test_xonsh.py +++ b/tests/unit/activation/test_xonsh.py @@ -3,6 +3,7 @@ import sys import pytest +from flaky import flaky from virtualenv.activation import XonshActivator from virtualenv.info import IS_PYPY, PY3 @@ -13,6 +14,7 @@ (sys.platform == "win32" and IS_PYPY and PY3) or sys.version_info[0:2] == (3, 9), reason="xonsh on Windows blocks indefinitely and is not stable yet on 3.9", ) +@flaky(max_runs=2, min_passes=1) def test_xonsh(activation_tester_class, activation_tester): class Xonsh(activation_tester_class): def __init__(self, session): diff --git a/tests/unit/config/test___main__.py b/tests/unit/config/test___main__.py index 91fef2774..3aad47594 100644 --- a/tests/unit/config/test___main__.py +++ b/tests/unit/config/test___main__.py @@ -1,7 +1,12 @@ from __future__ import absolute_import, unicode_literals +import re import sys +import pytest + +from virtualenv.__main__ import run_with_catch +from virtualenv.util.error import ProcessCallFailed from virtualenv.util.subprocess import Popen, subprocess @@ -10,3 +15,68 @@ def test_main(): out, _ = process.communicate() assert not process.returncode assert out + + +@pytest.fixture() +def raise_on_session_done(mocker): + def _func(exception): + from virtualenv.run import session_via_cli + + prev_session = session_via_cli + + def _session_via_cli(args, options=None): + prev_session(args, options) + raise exception + + mocker.patch("virtualenv.run.session_via_cli", side_effect=_session_via_cli) + + return _func + + +def test_fail_no_traceback(raise_on_session_done, tmp_path, capsys): + raise_on_session_done(ProcessCallFailed(code=2, out="out\n", err="err\n", cmd=["something"])) + with pytest.raises(SystemExit) as context: + run_with_catch([str(tmp_path)]) + assert context.value.code == 2 + out, err = capsys.readouterr() + assert out == "subprocess call failed for [{}] with code 2\nout\nSystemExit: 2\n".format(repr("something")) + assert err == "err\n" + + +def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys): + raise_on_session_done(TypeError("something bad")) + + with pytest.raises(TypeError, match="something bad"): + run_with_catch([str(tmp_path), "--with-traceback"]) + out, err = capsys.readouterr() + assert out == "" + assert err == "" + + +def test_session_report_full(session_app_data, tmp_path, capsys): + run_with_catch([str(tmp_path)]) + out, err = capsys.readouterr() + assert err == "" + lines = out.splitlines() + regexes = [ + r"created virtual environment .* in \d+ms", + r" creator .*", + r" seeder .*", + r" added seed packages: .*pip==.*, setuptools==.*, wheel==.*", + r" activators .*", + ] + for line, regex in zip(lines, regexes): + assert re.match(regex, line), line + + +def test_session_report_minimal(session_app_data, tmp_path, capsys): + run_with_catch([str(tmp_path), "--activators", "", "--without-pip"]) + out, err = capsys.readouterr() + assert err == "" + lines = out.splitlines() + regexes = [ + r"created virtual environment .* in \d+ms", + r" creator .*", + ] + for line, regex in zip(lines, regexes): + assert re.match(regex, line), line diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py index 1b9be2371..f51dcc226 100644 --- a/tests/unit/create/conftest.py +++ b/tests/unit/create/conftest.py @@ -30,7 +30,7 @@ def venv(tmp_path_factory, session_app_data): if CURRENT.is_venv: return sys.executable elif CURRENT.version_info.major == 3: - root_python = root(tmp_path_factory) + root_python = root(tmp_path_factory, session_app_data) dest = tmp_path_factory.mktemp("venv") process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)]) process.communicate() @@ -45,7 +45,7 @@ def old_virtualenv(tmp_path_factory, session_app_data): return CURRENT.executable else: env_for_old_virtualenv = tmp_path_factory.mktemp("env-for-old-virtualenv") - result = cli_run(["--no-download", "--activators", "", str(env_for_old_virtualenv)]) + result = cli_run(["--no-download", "--activators", "", str(env_for_old_virtualenv), "--no-periodic-update"]) # noinspection PyBroadException try: process = Popen( diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index be1f21f7d..f92808883 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -20,12 +20,12 @@ from virtualenv.__main__ import run, run_with_catch from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info +from virtualenv.create.pyenv_cfg import PyEnvCfg from virtualenv.create.via_global_ref.builtin.cpython.cpython2 import CPython2PosixBase from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix from virtualenv.create.via_global_ref.builtin.python2.python2 import Python2 from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_PYPY, IS_WIN, PY2, PY3, fs_is_case_sensitive -from virtualenv.pyenv_cfg import PyEnvCfg from virtualenv.run import cli_run, session_via_cli from virtualenv.util.path import Path from virtualenv.util.six import ensure_str, ensure_text @@ -48,7 +48,7 @@ def _non_success_exit_code(capsys, target): run_with_catch(args=[target]) assert context.value.code != 0 out, err = capsys.readouterr() - assert not out, out + assert "SystemExit: " in out return err @@ -384,7 +384,7 @@ def test_create_long_path(current_fastest, tmp_path): @pytest.mark.parametrize("creator", sorted(set(PythonInfo.current_system().creators().key_to_class) - {"builtin"})) -def test_create_distutils_cfg(creator, tmp_path, monkeypatch): +def test_create_distutils_cfg(creator, tmp_path, monkeypatch, session_app_data): result = cli_run([ensure_text(str(tmp_path / "venv")), "--activators", "", "--creator", creator]) app = Path(__file__).parent / "console_app" diff --git a/tests/unit/seed/greet/greet2.c b/tests/unit/create/via_global_ref/greet/greet2.c similarity index 100% rename from tests/unit/seed/greet/greet2.c rename to tests/unit/create/via_global_ref/greet/greet2.c diff --git a/tests/unit/seed/greet/greet3.c b/tests/unit/create/via_global_ref/greet/greet3.c similarity index 100% rename from tests/unit/seed/greet/greet3.c rename to tests/unit/create/via_global_ref/greet/greet3.c diff --git a/tests/unit/seed/greet/setup.py b/tests/unit/create/via_global_ref/greet/setup.py similarity index 100% rename from tests/unit/seed/greet/setup.py rename to tests/unit/create/via_global_ref/greet/setup.py diff --git a/tests/unit/seed/test_extra_install.py b/tests/unit/create/via_global_ref/test_build_c_ext.py similarity index 100% rename from tests/unit/seed/test_extra_install.py rename to tests/unit/create/via_global_ref/test_build_c_ext.py diff --git a/tests/unit/seed/test_base_embed.py b/tests/unit/seed/embed/test_base_embed.py similarity index 100% rename from tests/unit/seed/test_base_embed.py rename to tests/unit/seed/embed/test_base_embed.py diff --git a/tests/unit/seed/test_boostrap_link_via_app_data.py b/tests/unit/seed/embed/test_boostrap_link_via_app_data.py similarity index 95% rename from tests/unit/seed/test_boostrap_link_via_app_data.py rename to tests/unit/seed/embed/test_boostrap_link_via_app_data.py index 8f4704f5c..a48edd47e 100644 --- a/tests/unit/seed/test_boostrap_link_via_app_data.py +++ b/tests/unit/seed/embed/test_boostrap_link_via_app_data.py @@ -9,8 +9,7 @@ from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink from virtualenv.run import cli_run -from virtualenv.seed.embed.wheels import BUNDLE_SUPPORT -from virtualenv.seed.embed.wheels.acquire import BUNDLE_FOLDER +from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT from virtualenv.util.six import ensure_text from virtualenv.util.subprocess import Popen @@ -107,7 +106,7 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies) @pytest.fixture() -def read_only_folder(temp_app_data): +def read_only_app_data(temp_app_data): temp_app_data.mkdir() try: os.chmod(str(temp_app_data), S_IREAD | S_IRGRP | S_IROTH) @@ -117,7 +116,7 @@ def read_only_folder(temp_app_data): @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") -def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest, read_only_folder, monkeypatch): +def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest, read_only_app_data, monkeypatch): dest = tmp_path / "venv" result = cli_run(["--seeder", "app-data", "--creator", current_fastest, "--reset-app-data", "-vv", str(dest)]) assert result diff --git a/tests/unit/seed/embed/test_pip_invoke.py b/tests/unit/seed/embed/test_pip_invoke.py new file mode 100644 index 000000000..4c478dfe1 --- /dev/null +++ b/tests/unit/seed/embed/test_pip_invoke.py @@ -0,0 +1,89 @@ +from __future__ import absolute_import, unicode_literals + +import itertools +import sys +from shutil import copy2 + +import pytest + +from virtualenv.run import cli_run +from virtualenv.seed.embed.pip_invoke import PipInvoke +from virtualenv.seed.wheels.bundle import load_embed_wheel +from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT + + +@pytest.mark.slow +@pytest.mark.parametrize("no", ["pip", "setuptools", "wheel", ""]) +def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no): + extra_search_dir = tmp_path / "extra" + extra_search_dir.mkdir() + for_py_version = "{}.{}".format(*sys.version_info[0:2]) + new = BUNDLE_SUPPORT[for_py_version] + for wheel_filename in BUNDLE_SUPPORT[for_py_version].values(): + copy2(str(BUNDLE_FOLDER / wheel_filename), str(extra_search_dir)) + + def _load_embed_wheel(app_data, distribution, for_py_version, version): + return load_embed_wheel(app_data, distribution, old_ver, version) + + old_ver = "3.4" + old = BUNDLE_SUPPORT[old_ver] + mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", side_effect=_load_embed_wheel) + + def _execute(cmd, env): + expected = set() + for distribution, with_version in versions.items(): + if distribution == no: + continue + if with_version == "embed": + expected.add(BUNDLE_FOLDER) + elif old[dist] != new[dist]: + expected.add(extra_search_dir) + expected_list = list( + itertools.chain.from_iterable(["--find-links", str(e)] for e in sorted(expected, key=lambda x: str(x))), + ) + found = cmd[-len(expected_list) :] + assert "--no-index" not in cmd + cmd.append("--no-index") + assert found == expected_list + return original(cmd, env) + + original = PipInvoke._execute + run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute) + versions = {"pip": "embed", "setuptools": "bundle", "wheel": new["wheel"].split("-")[1]} + + create_cmd = [ + "--seeder", + "pip", + str(tmp_path / "env"), + "--download", + "--creator", + current_fastest, + "--extra-search-dir", + str(extra_search_dir), + "--app-data", + str(tmp_path / "app-data"), + ] + for dist, version in versions.items(): + create_cmd.extend(["--{}".format(dist), version]) + if no: + create_cmd.append("--no-{}".format(no)) + result = cli_run(create_cmd) + coverage_env() + + assert result + assert run.call_count == 1 + + site_package = result.creator.purelib + pip = site_package / "pip" + setuptools = site_package / "setuptools" + wheel = site_package / "wheel" + files_post_first_create = list(site_package.iterdir()) + + if no: + no_file = locals()[no] + assert no not in files_post_first_create + + for key in ("pip", "setuptools", "wheel"): + if key == no: + continue + assert locals()[key] in files_post_first_create diff --git a/tests/unit/seed/embed/wheels/test_acquire.py b/tests/unit/seed/embed/wheels/test_acquire.py deleted file mode 100644 index 49f743336..000000000 --- a/tests/unit/seed/embed/wheels/test_acquire.py +++ /dev/null @@ -1,11 +0,0 @@ -from virtualenv.seed.embed.wheels.acquire import get_bundled_wheel, wheel_support_py - - -def test_wheel_support_no_python_requires(mocker): - wheel = get_bundled_wheel(package="setuptools", version_release=None) - zip_mock = mocker.MagicMock() - mocker.patch("virtualenv.seed.embed.wheels.acquire.ZipFile", new=zip_mock) - zip_mock.return_value.__enter__.return_value.read = lambda name: b"" - - supports = wheel_support_py(wheel, "3.8") - assert supports is True diff --git a/tests/unit/seed/test_pip_invoke.py b/tests/unit/seed/test_pip_invoke.py deleted file mode 100644 index 65314aaab..000000000 --- a/tests/unit/seed/test_pip_invoke.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import absolute_import, unicode_literals - -import pytest - -from virtualenv.discovery.py_info import PythonInfo -from virtualenv.run import cli_run -from virtualenv.seed.embed.pip_invoke import PipInvoke -from virtualenv.seed.embed.wheels import BUNDLE_SUPPORT, MAX -from virtualenv.seed.embed.wheels.acquire import BUNDLE_FOLDER - - -@pytest.mark.slow -@pytest.mark.parametrize("no", ["pip", "setuptools", "wheel", ""]) -def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no): - bundle_ver = BUNDLE_SUPPORT.get(PythonInfo.current_system().version_release_str) or BUNDLE_SUPPORT.get(MAX) - - extra_search_dir = tmp_path / "extra" - extra_search_dir.mkdir() - - original = PipInvoke._execute - - def _execute(cmd, env): - assert set(cmd[-4:]) == {"--find-links", str(extra_search_dir), str(BUNDLE_FOLDER)} - return original(cmd, env) - - run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute) - - create_cmd = [ - "--seeder", - "pip", - str(tmp_path / "env"), - "--download", - "--pip", - bundle_ver["pip"].split("-")[1], - "--setuptools", - bundle_ver["setuptools"].split("-")[1], - "--creator", - current_fastest, - "--extra-search-dir", - str(extra_search_dir), - ] - if no: - create_cmd.append("--no-{}".format(no)) - result = cli_run(create_cmd) - coverage_env() - - assert result - assert run.call_count == 1 - - site_package = result.creator.purelib - pip = site_package / "pip" - setuptools = site_package / "setuptools" - wheel = site_package / "wheel" - files_post_first_create = list(site_package.iterdir()) - - if no: - no_file = locals()[no] - assert no not in files_post_first_create - - for key in ("pip", "setuptools", "wheel"): - if key == no: - continue - assert locals()[key] in files_post_first_create diff --git a/tests/unit/seed/wheels/test_acquire.py b/tests/unit/seed/wheels/test_acquire.py new file mode 100644 index 000000000..944ed4cea --- /dev/null +++ b/tests/unit/seed/wheels/test_acquire.py @@ -0,0 +1,66 @@ +from __future__ import absolute_import, unicode_literals + +import sys +from subprocess import CalledProcessError + +import pytest + +from virtualenv.info import PY2 +from virtualenv.seed.wheels.acquire import download_wheel, pip_wheel_env_run +from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, get_embed_wheel +from virtualenv.seed.wheels.util import discover_wheels + + +def test_pip_wheel_env_run_could_not_find(session_app_data, mocker): + mocker.patch("virtualenv.seed.wheels.acquire.from_bundle", return_value=None) + with pytest.raises(RuntimeError, match="could not find the embedded pip"): + pip_wheel_env_run([], session_app_data) + + +def test_download_wheel_bad_output(mocker, for_py_version, session_app_data): + """if the download contains no match for what wheel was downloaded, pick one that matches from target""" + distribution = "setuptools" + p_open = mocker.MagicMock() + mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open) + p_open.communicate.return_value = "", "" + p_open.returncode = 0 + + embed = get_embed_wheel(distribution, for_py_version) + as_path = mocker.MagicMock() + available = discover_wheels(BUNDLE_FOLDER, "setuptools", None, for_py_version) + as_path.iterdir.return_value = [i.path for i in available] + + result = download_wheel(distribution, "=={}".format(embed.version), for_py_version, [], session_app_data, as_path) + assert result.path == embed.path + + +def test_download_fails(mocker, for_py_version, session_app_data): + p_open = mocker.MagicMock() + mocker.patch("virtualenv.seed.wheels.acquire.Popen", return_value=p_open) + p_open.communicate.return_value = "out", "err" + p_open.returncode = 1 + + as_path = mocker.MagicMock() + with pytest.raises(CalledProcessError) as context: + download_wheel("pip", "==1", for_py_version, [], session_app_data, as_path), + exc = context.value + if PY2: + assert exc.output == "outerr" + else: + assert exc.output == "out" + assert exc.stderr == "err" + assert exc.returncode == 1 + assert [ + sys.executable, + "-m", + "pip", + "download", + "--disable-pip-version-check", + "--only-binary=:all:", + "--no-deps", + "--python-version", + for_py_version, + "-d", + str(as_path), + "pip==1", + ] == exc.cmd diff --git a/tests/unit/seed/wheels/test_acquire_find_wheel.py b/tests/unit/seed/wheels/test_acquire_find_wheel.py new file mode 100644 index 000000000..18c46927b --- /dev/null +++ b/tests/unit/seed/wheels/test_acquire_find_wheel.py @@ -0,0 +1,30 @@ +from __future__ import absolute_import, unicode_literals + +import pytest + +from virtualenv.seed.wheels.acquire import find_compatible_in_house +from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, MAX, get_embed_wheel + + +def test_find_latest(for_py_version): + result = find_compatible_in_house("setuptools", None, for_py_version, BUNDLE_FOLDER) + expected = get_embed_wheel("setuptools", for_py_version) + assert result.path == expected.path + + +def test_find_exact(for_py_version): + expected = get_embed_wheel("setuptools", for_py_version) + result = find_compatible_in_house("setuptools", "=={}".format(expected.version), for_py_version, BUNDLE_FOLDER) + assert result.path == expected.path + + +def test_find_less_than(for_py_version): + latest = get_embed_wheel("setuptools", MAX) + result = find_compatible_in_house("setuptools", "<{}".format(latest.version), MAX, BUNDLE_FOLDER) + assert result is not None + assert result.path != latest.path + + +def test_find_bad_spec(for_py_version): + with pytest.raises(ValueError, match="bad"): + find_compatible_in_house("setuptools", "bad", MAX, BUNDLE_FOLDER) diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py new file mode 100644 index 000000000..3d9507e86 --- /dev/null +++ b/tests/unit/seed/wheels/test_periodic_update.py @@ -0,0 +1,351 @@ +from __future__ import absolute_import, unicode_literals + +import json +import subprocess +import sys +from contextlib import contextmanager +from datetime import datetime, timedelta + +import pytest +from six import StringIO +from six.moves import zip_longest + +from virtualenv import cli_run +from virtualenv.app_data import AppDataDiskFolder +from virtualenv.seed.wheels import Wheel +from virtualenv.seed.wheels.embed import BUNDLE_SUPPORT, get_embed_wheel +from virtualenv.seed.wheels.periodic_update import ( + NewVersion, + UpdateLog, + do_update, + dump_datetime, + load_datetime, + manual_upgrade, + periodic_update, + trigger_update, +) +from virtualenv.util.path import Path +from virtualenv.util.subprocess import DETACHED_PROCESS + + +def test_manual_upgrade(session_app_data, caplog, mocker, for_py_version): + wheel = get_embed_wheel("pip", for_py_version) + new_version = NewVersion(wheel.path, datetime.now(), datetime.now() - timedelta(days=20)) + + def _do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): # noqa + if distribution == "pip": + return [new_version] + return [] + + do_update = mocker.patch("virtualenv.seed.wheels.periodic_update.do_update", side_effect=_do_update) + manual_upgrade(session_app_data) + + assert "upgrade pip" in caplog.text + assert "upgraded pip" in caplog.text + assert " new entries found:\n\tNewVersion" in caplog.text + assert " no new versions found" in caplog.text + assert do_update.call_count == 3 * len(BUNDLE_SUPPORT) + + +def test_pick_periodic_update(tmp_path, session_app_data, mocker, for_py_version): + embed, current = get_embed_wheel("setuptools", "3.4"), get_embed_wheel("setuptools", for_py_version) + mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", return_value=embed) + completed = datetime.now() - timedelta(days=29) + u_log = UpdateLog( + started=datetime.now() - timedelta(days=30), + completed=completed, + versions=[NewVersion(filename=current.path, found_date=completed, release_date=completed)], + periodic=True, + ) + read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + + result = cli_run([str(tmp_path), "--activators", "", "--no-periodic-update", "--no-wheel", "--no-pip"]) + + assert read_dict.call_count == 1 + installed = list(i.name for i in result.creator.purelib.iterdir() if i.suffix == ".dist-info") + assert "setuptools-{}.dist-info".format(current.version) in installed + + +def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_version): + current = get_embed_wheel("setuptools", for_py_version) + + now, completed = datetime.now(), datetime.now() - timedelta(days=29) + u_log = UpdateLog( + started=completed, + completed=completed, + versions=[ + NewVersion(wheel_path(current, (1,)), completed, now - timedelta(days=1)), + NewVersion(filename=current.path, found_date=completed, release_date=now - timedelta(days=2)), + NewVersion(wheel_path(current, (-1,)), completed, now - timedelta(days=30)), + ], + periodic=True, + ) + mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + + result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False) + assert result.path == current.path + + +def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_version): + current = get_embed_wheel("setuptools", for_py_version) + now, completed = datetime.now(), datetime.now() - timedelta(days=29) + u_log = UpdateLog( + started=completed, + completed=completed, + versions=[ + NewVersion(wheel_path(current, (0, 1, 2)), completed, now - timedelta(days=1)), + NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=30)), + NewVersion(filename=str(current.path), found_date=completed, release_date=now - timedelta(days=2)), + ], + periodic=True, + ) + mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + + result = periodic_update("setuptools", for_py_version, current, [], session_app_data, False) + assert result.path == current.path + + +def wheel_path(wheel, of): + new_version = ".".join(str(i) for i in (tuple(sum(x) for x in zip_longest(wheel.version_tuple, of, fillvalue=0)))) + new_name = wheel.name.replace(wheel.version, new_version) + return str(wheel.path.parent / new_name) + + +_UP_NOW = datetime.now() +_UPDATE_SKIP = { + "started_just_now_no_complete": UpdateLog(started=_UP_NOW, completed=None, versions=[], periodic=True), + "started_1_hour_no_complete": UpdateLog( + started=_UP_NOW - timedelta(hours=1), completed=None, versions=[], periodic=True, + ), + "completed_under_two_weeks": UpdateLog( + started=None, completed=_UP_NOW - timedelta(days=14), versions=[], periodic=True, + ), + "started_just_now_completed_two_weeks": UpdateLog( + started=_UP_NOW, completed=_UP_NOW - timedelta(days=14, seconds=1), versions=[], periodic=True, + ), + "started_1_hour_completed_two_weeks": UpdateLog( + started=_UP_NOW - timedelta(hours=1), + completed=_UP_NOW - timedelta(days=14, seconds=1), + versions=[], + periodic=True, + ), +} + + +@pytest.mark.parametrize("u_log", _UPDATE_SKIP.values(), ids=_UPDATE_SKIP.keys()) +def test_periodic_update_skip(u_log, mocker, for_py_version, session_app_data, freezer): + freezer.move_to(_UP_NOW) + mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update", side_effect=RuntimeError) + + result = periodic_update("setuptools", for_py_version, None, [], session_app_data, True) + assert result is None + + +_UPDATE_YES = { + "never_started": UpdateLog(started=None, completed=None, versions=[], periodic=False), + "started_1_hour": UpdateLog( + started=_UP_NOW - timedelta(hours=1, microseconds=1), completed=None, versions=[], periodic=False, + ), + "completed_two_week": UpdateLog( + started=_UP_NOW - timedelta(days=14, microseconds=2), + completed=_UP_NOW - timedelta(days=14, microseconds=1), + versions=[], + periodic=False, + ), +} + + +@pytest.mark.parametrize("u_log", _UPDATE_YES.values(), ids=_UPDATE_YES.keys()) +def test_periodic_update_trigger(u_log, mocker, for_py_version, session_app_data, freezer): + freezer.move_to(_UP_NOW) + mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") + trigger_update_ = mocker.patch("virtualenv.seed.wheels.periodic_update.trigger_update") + + result = periodic_update("setuptools", for_py_version, None, [], session_app_data, True) + + assert result is None + assert trigger_update_.call_count + assert write.call_count == 1 + wrote_json = write.call_args[0][0] + assert wrote_json["periodic"] is True + assert load_datetime(wrote_json["started"]) == _UP_NOW + + +def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch): + monkeypatch.delenv(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE"), raising=False) + current = get_embed_wheel("setuptools", for_py_version) + process = mocker.MagicMock() + process.communicate.return_value = None, None + Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) + + trigger_update("setuptools", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, True) + + assert Popen.call_count == 1 + args, kwargs = Popen.call_args + cmd = ( + "from virtualenv.seed.wheels.periodic_update import do_update;" + "do_update({!r}, {!r}, {!r}, {!r}, [{!r}, {!r}], True)".format( + "setuptools", + for_py_version, + str(current.path), + str(session_app_data), + str(tmp_path / "a"), + str(tmp_path / "b"), + ) + ) + assert args == ([sys.executable, "-c", cmd],) + expected = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE} + if sys.platform == "win32": + expected["creationflags"] = DETACHED_PROCESS + assert kwargs == expected + assert process.communicate.call_count == 0 + + +def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker, monkeypatch): + monkeypatch.setenv(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE"), str("1")) + current = get_embed_wheel("pip", for_py_version) + + process = mocker.MagicMock() + process.communicate.return_value = None, None + Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) + + trigger_update("pip", for_py_version, current, [tmp_path / "a", tmp_path / "b"], session_app_data, False) + + assert Popen.call_count == 1 + args, kwargs = Popen.call_args + cmd = ( + "from virtualenv.seed.wheels.periodic_update import do_update;" + "do_update({!r}, {!r}, {!r}, {!r}, [{!r}, {!r}], False)".format( + "pip", for_py_version, str(current.path), str(session_app_data), str(tmp_path / "a"), str(tmp_path / "b"), + ) + ) + assert args == ([sys.executable, "-c", cmd],) + expected = {"stdout": None, "stderr": None} + assert kwargs == expected + assert process.communicate.call_count == 1 + + +def test_do_update_first(tmp_path, mocker, freezer): + freezer.move_to(_UP_NOW) + wheel = get_embed_wheel("pip", "3.9") + app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) + extra = tmp_path / "extra" + extra.mkdir() + + pip_version_remote = [ + (wheel_path(wheel, (1, 0, 0)), None), + (wheel_path(wheel, (0, 1, 0)), _UP_NOW - timedelta(days=1)), + (wheel_path(wheel, (0, 0, 1)), _UP_NOW - timedelta(days=2)), + (wheel.path, _UP_NOW - timedelta(days=3)), + (wheel_path(wheel, (-1, 0, 0)), _UP_NOW - timedelta(days=30)), + ] + download_wheels = (Wheel(Path(i[0])) for i in pip_version_remote) + + def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder): + assert distribution == "pip" + assert for_py_version == "3.9" + assert [str(i) for i in search_dirs] == [str(extra)] + assert isinstance(app_data, AppDataDiskFolder) + assert to_folder == app_data_outer.house + return next(download_wheels) + + download_wheel = mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=_download_wheel) + releases = { + Wheel(Path(wheel)).version: [ + {"upload_time": datetime.strftime(release_date, "%Y-%m-%dT%H:%M:%S") if release_date is not None else None}, + ] + for wheel, release_date in pip_version_remote + } + pypi_release = json.dumps({"releases": releases}) + + @contextmanager + def _release(of): + assert of == "https://pypi.org/pypi/pip/json" + yield StringIO(pypi_release) + + url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=_release) + + last_update = _UP_NOW - timedelta(days=14) + u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True) + read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") + + versions = do_update("pip", "3.9", str(pip_version_remote[-1][0]), str(app_data_outer), [str(extra)], True) + + assert download_wheel.call_count == len(pip_version_remote) + assert url_o.call_count == len(pip_version_remote) + + expected = [ + NewVersion(Path(wheel).name, _UP_NOW, None if release is None else release.replace(microsecond=0)) + for wheel, release in pip_version_remote + ] + assert versions == expected + + assert read_dict.call_count == 1 + assert write.call_count == 1 + wrote_json = write.call_args[0][0] + assert wrote_json == { + "started": dump_datetime(last_update), + "completed": dump_datetime(_UP_NOW), + "periodic": True, + "versions": [e.to_dict() for e in expected], + } + + +def test_do_update_skip_already_done(tmp_path, mocker, freezer): + freezer.move_to(_UP_NOW + timedelta(hours=1)) + wheel = get_embed_wheel("pip", "3.9") + app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) + extra = tmp_path / "extra" + extra.mkdir() + + def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder): # noqa + return wheel.path + + download_wheel = mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", side_effect=_download_wheel) + url_o = mocker.patch("virtualenv.seed.wheels.periodic_update.urlopen", side_effect=RuntimeError) + + released = _UP_NOW - timedelta(days=30) + u_log = UpdateLog( + started=_UP_NOW - timedelta(days=31), + completed=released, + versions=[NewVersion(filename=wheel.path.name, found_date=released, release_date=released)], + periodic=True, + ) + read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") + + versions = do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [str(extra)], False) + + assert download_wheel.call_count == 1 + assert read_dict.call_count == 1 + assert not url_o.call_count + assert versions == [] + + assert write.call_count == 1 + wrote_json = write.call_args[0][0] + assert wrote_json == { + "started": dump_datetime(_UP_NOW + timedelta(hours=1)), + "completed": dump_datetime(_UP_NOW + timedelta(hours=1)), + "periodic": False, + "versions": [ + { + "filename": wheel.path.name, + "release_date": dump_datetime(released), + "found_date": dump_datetime(released), + }, + ], + } + + +def test_new_version_eq(): + value = NewVersion("a", datetime.now(), datetime.now()) + assert value == value + + +def test_new_version_ne(): + assert NewVersion("a", datetime.now(), datetime.now()) != NewVersion( + "a", datetime.now(), datetime.now() + timedelta(hours=1), + ) diff --git a/tests/unit/seed/wheels/test_wheels_util.py b/tests/unit/seed/wheels/test_wheels_util.py new file mode 100644 index 000000000..e487797bc --- /dev/null +++ b/tests/unit/seed/wheels/test_wheels_util.py @@ -0,0 +1,31 @@ +from __future__ import absolute_import, unicode_literals + +import pytest + +from virtualenv.seed.wheels.embed import MAX, get_embed_wheel +from virtualenv.seed.wheels.util import Wheel + + +def test_wheel_support_no_python_requires(mocker): + wheel = get_embed_wheel("setuptools", for_py_version=None) + zip_mock = mocker.MagicMock() + mocker.patch("virtualenv.seed.wheels.util.ZipFile", new=zip_mock) + zip_mock.return_value.__enter__.return_value.read = lambda name: b"" + + supports = wheel.support_py("3.8") + assert supports is True + + +def test_bad_as_version_tuple(): + with pytest.raises(ValueError, match="bad"): + Wheel.as_version_tuple("bad") + + +def test_wheel_not_support(): + wheel = get_embed_wheel("setuptools", MAX) + assert wheel.support_py("3.3") is False + + +def test_wheel_repr(): + wheel = get_embed_wheel("setuptools", MAX) + assert str(wheel.path) in repr(wheel) diff --git a/tox.ini b/tox.ini index 1570e68cc..0987acb90 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ setenv = COVERAGE_PROCESS_START = {toxinidir}/.coveragerc _COVERAGE_SRC = {envsitepackagesdir}/virtualenv PYTHONIOENCODING=utf-8 - {py34,py27,pypy}: PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command + {py34,py27,pypy, upgrade}: PYTHONWARNINGS=ignore:DEPRECATION::pip._internal.cli.base_command {pypy,py27}: PYTEST_XDIST = 0 passenv = https_proxy http_proxy no_proxy HOME PYTEST_* PIP_* CI_RUN TERM extras = testing @@ -37,7 +37,7 @@ commands = tests {posargs:--int --timeout 600 -n {env:PYTEST_XDIST:auto}} python -m coverage combine - python -m coverage report + python -m coverage report --skip-covered --show-missing python -m coverage xml -o {toxworkdir}/coverage.{envname}.xml python -m coverage html -d {envtmpdir}/htmlcov @@ -55,7 +55,7 @@ setenv = COVERAGE_FILE={toxworkdir}/.coverage commands = python -m coverage combine - python -m coverage report --show-missing + python -m coverage report --skip-covered --show-missing python -m coverage xml -o {toxworkdir}/coverage.xml python -m coverage html -d {toxworkdir}/htmlcov python -m diff_cover.diff_cover_tool --compare-branch {env:DIFF_AGAINST:origin/master} {toxworkdir}/coverage.xml