Implement periodic update feature (#1841)

Co-authored-by: Pradyun Gedam <[email protected]>

.coveragerc

@@ -1,5 +1,4 @@
 [coverage:report]
-skip_covered = True
 show_missing = True
 exclude_lines =
     \#\s*pragma: no cover
@@ -9,8 +8,10 @@ exclude_lines =
     ^if __name__ == ['"]__main__['"]:$
 omit =
     # site.py is ran before the coverage can be enabled, no way to measure coverage on this
-    src/virtualenv/interpreters/create/impl/cpython/site.py
-    src/virtualenv/seed/embed/wheels/pip-*.whl/*
+    src/virtualenv/create/via_global_ref/builtin/python2/site.py
+    src/virtualenv/create/via_global_ref/_virtualenv.py
+    src/virtualenv/activation/python/activate_this.py
+    src/virtualenv/seed/wheels/embed/pip-*.whl/*
 
 [coverage:paths]
 source =
@@ -24,5 +25,9 @@ source =
 [coverage:run]
 branch = false
 parallel = true
+dynamic_context = test_function
 source =
     ${_COVERAGE_SRC}
+
+[coverage:html]
+show_contexts = true

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v2.5.0
+  rev: v3.1.0
   hooks:
   - id: check-ast
   - id: check-builtin-literals
@@ -15,16 +15,12 @@ repos:
   rev: v2.0.1
   hooks:
   - id: add-trailing-comma
-- repo: https://github.com/asottile/yesqa
-  rev: v1.1.0
-  hooks:
-  - id: yesqa
 - repo: https://github.com/asottile/pyupgrade
-  rev: v2.4.1
+  rev: v2.6.1
   hooks:
   - id: pyupgrade
 - repo: https://github.com/asottile/seed-isort-config
-  rev: v2.1.1
+  rev: v2.2.0
   hooks:
   - id: seed-isort-config
     args: [--application-directories, '.:src']
@@ -54,8 +50,8 @@ repos:
   - id: setup-cfg-fmt
     args: [--min-py3-version, "3.4"]
 - repo: https://gitlab.com/pycqa/flake8
-  rev: "3.8.1"
+  rev: "3.8.3"
   hooks:
   - id: flake8
-    additional_dependencies: ["flake8-bugbear == 20.1.2"]
+    additional_dependencies: ["flake8-bugbear == 20.1.4"]
     language_version: python3.8

codecov.yaml

@@ -1,3 +1,8 @@
+ignore:
+- "src/virtualenv/create/via_global_ref/builtin/python2/site.py"
+- "src/virtualenv/create/via_global_ref/_virtualenv.py"
+- "src/virtualenv/activation/python/activate_this.py"
+
 coverage:
   range: "80...100"
   coverage:

docs/changelog/1821.doc.rst

@@ -0,0 +1 @@
+Document how bundled wheels are handled and (potentially automatically) kept up to date - by :user:`gaborbernat`.

docs/changelog/1821.feature.rst

@@ -0,0 +1,8 @@
+Better handling of bundled wheel installation:
+
+- display the installed seed package versions in the final summary output
+- add a manual upgrade of embedded wheels feature via :option:`upgrade-embed-wheels` CLI flag
+- periodically (once every 14 days) try to automatically upgrade the embedded wheels in the background, can be disabled
+  via :option:`no-periodic-update`
+
+by :user:`gaborbernat`.

docs/changelog/1841.feature.rst

@@ -0,0 +1,7 @@
+Bump embed wheel content:
+
+- ship wheels for Python ``3.9``
+- upgrade embedded setuptools for Python ``3.5+`` from ``46.4.0`` to ``47.1.1``
+- upgrade embedded setuptools for Python ``2.7`` from ``44.1.0`` to ``44.1.1``
+
+by :user:`gaborbernat`.

docs/cli_interface.rst

@@ -22,7 +22,7 @@ The options that can be passed to virtualenv, along with their default values an
 
 .. table_cli::
    :module: virtualenv.run
-   :func: build_parser
+   :func: build_parser_only
 
 Defaults
 ~~~~~~~~

docs/user_guide.rst

@@ -120,7 +120,7 @@ enables you to install additional python packages into the created virtual envir
 main seed mechanism available:
 
 - ``pip`` - this method uses the bundled pip with virtualenv to install the seed packages (note, a new child process
-  needs to be created to do this).
+  needs to be created to do this, which can be expensive especially on Windows).
 - ``app-data`` - this method uses the user application data directory to create install images. These images are needed
   to be created only once, and subsequent virtual environments can just link/copy those images into their pure python
   library path (the ``site-packages`` folder). This allows all but the first virtual environment creation to be blazing
@@ -131,6 +131,66 @@ main seed mechanism available:
   To override the filesystem location of the seed cache, one can use the
   ``VIRTUALENV_OVERRIDE_APP_DATA`` environment variable.
 
+Wheels
+~~~~~~
+
+To install a seed package via either ``pip`` or ``app-data`` method virtualenv needs to acquire a wheel of the target
+package. These wheels may be acquired from multiple locations as follows:
+
+- ``virtualenv`` ships out of box with a set of embed ``wheels`` for all three seed packages (:pypi:`pip`,
+  :pypi:`setuptools`, :pypi:`wheel`). These are packaged together with the virtualenv source files, and only change upon
+  upgrading virtualenv. Different Python versions require different versions of these, and because virtualenv supports a
+  wide range of Python versions, the number of embedded wheels out of box is greater than 3. Whenever newer versions of
+  these embedded packages are released upstream ``virtualenv`` project upgrades them, and does a new release. Therefore,
+  upgrading virtualenv periodically will also upgrade the version of the seed packages.
+- However, end users might not be able to upgrade virtualenv at the same speed as we do new releases. Therefore, a user
+  might request to upgrade the list of embedded wheels by invoking virtualenv with the :option:`upgrade-embed-wheels`
+  flag. If the operation is triggered in such manual way subsequent runs of virtualenv will always use the upgraded
+  embed wheels.
+
+  The operation can trigger automatically too, as a background process upon invocation of virtualenv, if no such upgrade
+  has been performed in the last 14 days. It will only start using automatically upgraded wheel if they have been
+  released for more than 28 days, and the automatic upgrade finished at least an hour ago:
+
+  - the 28 days period should guarantee end users are not pulling in automatically releases that have known bugs within,
+  - the one hour period after the automatic upgrade finished is implemented so that continuous integration services do
+    not start using a new embedded versions half way through.
+
+
+  The automatic behaviour might be disabled via the :option:`no-periodic-update` configuration flag/option. To acquire
+  the release date of a package virtualenv will perform the following:
+
+  - lookup ``https://pypi.org/pypi/<distribution>/json`` (primary truth source),
+  - save the date the version was first discovered, and wait until 28 days passed.
+- Users can specify a set of local paths containing additional wheels by using the :option:`extra-search-dir` command
+  line argument flag.
+
+When searching for a wheel to use virtualenv performs lookup in the following order:
+
+- embedded wheels,
+- upgraded embedded wheels,
+- extra search dir.
+
+Bundled wheels are all three above together. If neither of the locations contain the requested wheel version or
+:option:`download` option is set will use ``pip`` download to load the latest version available from the index server.
+
+Embed wheels for distributions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Custom distributions often want to use their own set of wheel versions to distribute instead of the one virtualenv
+releases on PyPi. The reason for this is trying to keep the system versions of those package in sync with what
+virtualenv uses. In such cases they should patch the module `virtualenv.seed.wheels.embed
+<https://github.com/pypa/virtualenv/tree/bundle/src/virtualenv/seed/wheels/embed>`_, making sure to provide the function
+``get_embed_wheel`` (which returns the wheel to use given a distribution/python version). The ``BUNDLE_FOLDER``,
+``BUNDLE_SUPPORT`` and ``MAX`` variables are needed if they want to use virtualenvs test suite to validate.
+
+Furthermore, they might want to disable the periodic update by patching the
+`virtualenv.seed.embed.base_embed.PERIODIC_UPDATE_ON_BY_DEFAULT
+<https://github.com/pypa/virtualenv/tree/bundle/src/virtualenv/seed/embed/base_embed.py>`_
+to ``False``, and letting the system update mechanism to handle this. Note in this case the user might still request an
+upgrade of the embedded wheels by invoking virtualenv via :option:`upgrade-embed-wheels`, but no longer happens
+automatically, and will not alter the OS provided wheels.
+
 Activators
 ----------
 These are activation scripts that will mangle with your shells settings to ensure that commands from within the python
@@ -201,7 +261,7 @@ about the created virtual environment.
 .. automodule:: virtualenv
    :members:
 
-.. currentmodule:: virtualenv.session
+.. currentmodule:: virtualenv.run.session
 
 .. autoclass:: Session
     :members:

setup.cfg

@@ -43,7 +43,6 @@ install_requires =
     distlib>=0.3.0,<1
     filelock>=3.0.0,<4
     six>=1.9.0,<2   # keep it >=1.9.0 as it may cause problems on LTS platforms
-    contextlib2>=0.6.0,<1;python_version<"3.3"
     importlib-metadata>=0.12,<2;python_version<"3.8"
     importlib-resources>=1.0;python_version<"3.7"
     pathlib2>=2.3.3,<3;python_version < '3.4' and sys.platform != 'win32'
@@ -79,7 +78,7 @@ virtualenv.discovery =
     builtin = virtualenv.discovery.builtin:Builtin
 virtualenv.seed =
     pip = virtualenv.seed.embed.pip_invoke:PipInvoke
-    app-data = virtualenv.seed.via_app_data.via_app_data:FromAppData
+    app-data = virtualenv.seed.embed.via_app_data.via_app_data:FromAppData
 
 [options.extras_require]
 docs =
@@ -98,6 +97,7 @@ testing =
     pytest-env >= 0.6.2
     pytest-randomly >= 1
     pytest-timeout >= 1
+    pytest-freezegun >= 0.4.1
     flaky >= 3
     xonsh >= 0.9.16; python_version > '3.4' and python_version != '3.9'
 
@@ -108,7 +108,7 @@ virtualenv.activation.cshell = *.csh
 virtualenv.activation.fish = *.fish
 virtualenv.activation.powershell = *.ps1
 virtualenv.activation.xonsh = *.xsh
-virtualenv.seed.embed.wheels = *.whl
+virtualenv.seed.wheels.embed = *.whl
 
 [options.packages.find]
 where = src

setup.py

@@ -4,6 +4,9 @@
     raise RuntimeError("setuptools >= 41 required to build")
 
 setup(
-    use_scm_version={"write_to": "src/virtualenv/version.py", "write_to_template": '__version__ = "{version}"'},
+    use_scm_version={
+        "write_to": "src/virtualenv/version.py",
+        "write_to_template": 'from __future__ import  unicode_literals;\n\n__version__ = "{version}"',
+    },
     setup_requires=["setuptools_scm >= 2"],
 )

src/virtualenv/__main__.py

@@ -5,13 +5,10 @@
 import sys
 from datetime import datetime
 
-from virtualenv.config.cli.parser import VirtualEnvOptions
-from virtualenv.util.six import ensure_text
-
 
 def run(args=None, options=None):
     start = datetime.now()
-    from virtualenv.error import ProcessCallFailed
+    from virtualenv.util.error import ProcessCallFailed
     from virtualenv.run import cli_run
 
     if args is None:
@@ -32,31 +29,49 @@ def __init__(self, session, start):
         self.start = start
 
     def __str__(self):
+        from virtualenv.util.six import ensure_text
+
         spec = self.session.creator.interpreter.spec
         elapsed = (datetime.now() - self.start).total_seconds() * 1000
         lines = [
             "created virtual environment {} in {:.0f}ms".format(spec, elapsed),
             "  creator {}".format(ensure_text(str(self.session.creator))),
         ]
         if self.session.seeder.enabled:
-            lines += ("  seeder {}".format(ensure_text(str(self.session.seeder))),)
+            lines += (
+                "  seeder {}".format(ensure_text(str(self.session.seeder))),
+                "    added seed packages: {}".format(
+                    ", ".join(
+                        sorted(
+                            "==".join(i.stem.split("-"))
+                            for i in self.session.creator.purelib.iterdir()
+                            if i.suffix == ".dist-info"
+                        ),
+                    ),
+                ),
+            )
         if self.session.activators:
             lines.append("  activators {}".format(",".join(i.__class__.__name__ for i in self.session.activators)))
         return os.linesep.join(lines)
 
 
 def run_with_catch(args=None):
+    from virtualenv.config.cli.parser import VirtualEnvOptions
+
     options = VirtualEnvOptions()
     try:
         run(args, options)
-    except (KeyboardInterrupt, Exception) as exception:
-        if getattr(options, "with_traceback", False):
+    except (KeyboardInterrupt, SystemExit, Exception) as exception:
+        try:
+            if getattr(options, "with_traceback", False):
+                raise
+            else:
+                logging.error("%s: %s", type(exception).__name__, exception)
+                code = exception.code if isinstance(exception, SystemExit) else 1
+                sys.exit(code)
+        finally:
             logging.shutdown()  # force flush of log messages before the trace is printed
-            raise
-        else:
-            logging.error("%s: %s", type(exception).__name__, exception)
-            sys.exit(1)
 
 
-if __name__ == "__main__":
-    run_with_catch()
+if __name__ == "__main__":  # pragma: no cov
+    run_with_catch()  # pragma: no cov

src/virtualenv/run/app_data.py → src/virtualenv/app_data/__init__.py

@@ -1,47 +1,25 @@
+"""
+Application data stored by virtualenv.
+"""
+from __future__ import absolute_import, unicode_literals
+
 import logging
 import os
 from argparse import Action, ArgumentError
-from tempfile import mkdtemp
 
 from appdirs import user_data_dir
 
-from virtualenv.util.lock import ReentrantFileLock
-from virtualenv.util.path import safe_delete
-
-
-class AppData(object):
-    def __init__(self, folder):
-        self.folder = ReentrantFileLock(folder)
-        self.transient = False
-
-    def __repr__(self):
-        return "{}".format(self.folder.path)
-
-    def clean(self):
-        logging.debug("clean app data folder %s", self.folder.path)
-        safe_delete(self.folder.path)
-
-    def close(self):
-        """"""
-
-
-class TempAppData(AppData):
-    def __init__(self):
-        super(TempAppData, self).__init__(folder=mkdtemp())
-        self.transient = True
-        logging.debug("created temporary app data folder %s", self.folder.path)
-
-    def close(self):
-        logging.debug("remove temporary app data folder %s", self.folder.path)
-        safe_delete(self.folder.path)
+from .na import AppDataDisabled
+from .via_disk_folder import AppDataDiskFolder
+from .via_tempdir import TempAppData
 
 
 class AppDataAction(Action):
     def __call__(self, parser, namespace, values, option_string=None):
         folder = self._check_folder(values)
         if folder is None:
             raise ArgumentError("app data path {} is not valid".format(values))
-        setattr(namespace, self.dest, AppData(folder))
+        setattr(namespace, self.dest, AppDataDiskFolder(folder))
 
     @staticmethod
     def _check_folder(folder):
@@ -64,8 +42,8 @@ def default():
         for folder in AppDataAction._app_data_candidates():
             folder = AppDataAction._check_folder(folder)
             if folder is not None:
-                return AppData(folder)
-        return None
+                return AppDataDiskFolder(folder)
+        return AppDataDisabled()
 
     @staticmethod
     def _app_data_candidates():
@@ -77,7 +55,8 @@ def _app_data_candidates():
 
 
 __all__ = (
-    "AppData",
+    "AppDataDiskFolder",
     "TempAppData",
     "AppDataAction",
+    "AppDataDisabled",
 )