Fix Windows Store support

docs/changelog/1663.bugfix.rst

@@ -0,0 +1 @@
+Support Python 3 Framework distributed via XCode in macOs Catalina and before - by :user:`gaborbernat`.

docs/changelog/1709.bugfix.rst

@@ -0,0 +1,2 @@
+Fix Windows Store Python support, do not allow creation via symlink as that's not going to work by design
+- by :user:`gaborbernat`.

docs/changelog/1716.feature.rst

@@ -0,0 +1,5 @@
+Improve error message when the host python does not satisfy invariants needed to create virtual environments (now we
+print which host files are incompatible/missing and for which creators when no supported creator can be matched, however
+we found creators that can describe the given Python interpreter - will still print no supported creator for Jython,
+but print exactly what host files do not allow creation of virtual environments in case of CPython/PyPy)
+- by :user:`gaborbernat`.

docs/installation.rst

@@ -116,8 +116,7 @@ In case of macOs we support:
 Windows
 ~~~~~~~
 - Installations from `python.org <https://www.python.org/downloads/>`_
-- Windows Store Python - note only `version 3.8+ <https://www.microsoft.com/en-us/p/python-38/9mssztt1n39l>`_ (``3.7``
-  was marked experimental and contains many bugs that would make it very hard for us to support it)
+- Windows Store Python - note only `version 3.7+ <https://www.microsoft.com/en-us/p/python-38/9mssztt1n39l>`_
 
 Packaging variants
 ~~~~~~~~~~~~~~~~~~

docs/render_cli.py

@@ -76,12 +76,16 @@ def a(*args, **kwargs):
                         return True
                     elif key == "creator":
                         if name == "venv":
-                            from virtualenv.create.via_global_ref.venv import Meta
+                            from virtualenv.create.via_global_ref.venv import ViaGlobalRefMeta
 
-                            return Meta(True, True)
-                        from virtualenv.create.via_global_ref.builtin.via_global_self_do import Meta
+                            meta = ViaGlobalRefMeta()
+                            meta.symlink_error = None
+                            return meta
+                        from virtualenv.create.via_global_ref.builtin.via_global_self_do import BuiltinViaGlobalRefMeta
 
-                        return Meta([], True, True)
+                        meta = BuiltinViaGlobalRefMeta()
+                        meta.symlink_error = None
+                        return meta
                     raise RuntimeError
 
                 setattr(class_n, func_name, a)

src/virtualenv/__main__.py

@@ -20,7 +20,7 @@ def run(args=None, options=None):
         session = cli_run(args, options)
         logging.warning(LogSession(session, start))
     except ProcessCallFailed as exception:
-        print("subprocess call failed for {}".format(exception.cmd))
+        print("subprocess call failed for {} with code {}".format(exception.cmd, exception.code))
         print(exception.out, file=sys.stdout, end="")
         print(exception.err, file=sys.stderr, end="")
         raise SystemExit(exception.code)

src/virtualenv/create/creator.py

@@ -24,6 +24,11 @@
 DEBUG_SCRIPT = HERE / "debug.py"
 
 
+class CreatorMeta(object):
+    def __init__(self):
+        self.error = None
+
+
 @add_metaclass(ABCMeta)
 class Creator(object):
     """A class that given a python Interpreter creates a virtual environment"""

src/virtualenv/create/via_global_ref/api.py

@@ -6,10 +6,28 @@
 
 from six import add_metaclass
 
+from virtualenv.info import fs_supports_symlink
 from virtualenv.util.path import Path
 from virtualenv.util.zipapp import ensure_file_on_disk
 
-from ..creator import Creator
+from ..creator import Creator, CreatorMeta
+
+
+class ViaGlobalRefMeta(CreatorMeta):
+    def __init__(self):
+        super(ViaGlobalRefMeta, self).__init__()
+        self.copy_error = None
+        self.symlink_error = None
+        if not fs_supports_symlink():
+            self.symlink = "the filesystem does not supports symlink"
+
+    @property
+    def can_copy(self):
+        return not self.copy_error
+
+    @property
+    def can_symlink(self):
+        return not self.symlink_error
 
 
 @add_metaclass(ABCMeta)

src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py

@@ -6,6 +6,7 @@
 
 from virtualenv.create.describe import Python3Supports
 from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.create.via_global_ref.store import is_store_python
 from virtualenv.util.path import Path
 
 from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework
@@ -25,6 +26,12 @@ def can_describe(cls, interpreter):
 class CPython3Windows(CPythonWindows, CPython3):
     """"""
 
+    @classmethod
+    def setup_meta(cls, interpreter):
+        if is_store_python(interpreter):  # store python is not supported here
+            return None
+        return super(CPython3Windows, cls).setup_meta(interpreter)
+
     @classmethod
     def sources(cls, interpreter):
         for src in super(CPython3Windows, cls).sources(interpreter):

src/virtualenv/create/via_global_ref/builtin/ref.py

@@ -28,7 +28,10 @@ def __init__(self, src, must_symlink, must_copy):
         self.must_symlink = must_symlink
         self.must_copy = must_copy
         self.src = src
-        self.exists = src.exists()
+        try:
+            self.exists = src.exists()
+        except OSError:
+            self.exists = False
         self._can_read = None if self.exists else False
         self._can_copy = None if self.exists else False
         self._can_symlink = None if self.exists else False
@@ -141,7 +144,8 @@ def run(self, creator, symlinks):
         dest = bin_dir / self.base
         method = self.method(symlinks)
         method(self.src, dest)
-        make_exe(dest)
+        if not symlinks:
+            make_exe(dest)
         for extra in self.aliases:
             link_file = bin_dir / extra
             if link_file.exists():
@@ -150,4 +154,8 @@ def run(self, creator, symlinks):
                 link_file.symlink_to(self.base)
             else:
                 copy(self.src, link_file)
-            make_exe(link_file)
+            if not symlinks:
+                make_exe(link_file)
+
+    def __repr__(self):
+        return "{}(src={}, alias={})".format(self.__class__.__name__, self.src, self.aliases)

src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py

@@ -1,19 +1,20 @@
 from __future__ import absolute_import, unicode_literals
 
-import logging
 from abc import ABCMeta
-from collections import namedtuple
 
 from six import add_metaclass
 
 from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest
-from virtualenv.info import fs_supports_symlink
 from virtualenv.util.path import ensure_dir
 
-from ..api import ViaGlobalRefApi
+from ..api import ViaGlobalRefApi, ViaGlobalRefMeta
 from .builtin_way import VirtualenvBuiltin
 
-Meta = namedtuple("Meta", ["sources", "can_copy", "can_symlink"])
+
+class BuiltinViaGlobalRefMeta(ViaGlobalRefMeta):
+    def __init__(self):
+        super(BuiltinViaGlobalRefMeta, self).__init__()
+        self.sources = []
 
 
 @add_metaclass(ABCMeta)
@@ -27,27 +28,30 @@ def can_create(cls, interpreter):
         """By default all built-in methods assume that if we can describe it we can create it"""
         # first we must be able to describe it
         if cls.can_describe(interpreter):
-            sources = []
-            can_copy = True
-            can_symlink = fs_supports_symlink()
-            for src in cls.sources(interpreter):
-                if src.exists:
-                    if can_copy and not src.can_copy:
-                        can_copy = False
-                        logging.debug("%s cannot copy %s", cls.__name__, src)
-                    if can_symlink and not src.can_symlink:
-                        can_symlink = False
-                        logging.debug("%s cannot symlink %s", cls.__name__, src)
-                    if not (can_copy or can_symlink):
+            meta = cls.setup_meta(interpreter)
+            if meta is not None and meta:
+                for src in cls.sources(interpreter):
+                    if src.exists:
+                        if meta.can_copy and not src.can_copy:
+                            meta.copy_error = "cannot copy {}".format(src)
+                        if meta.can_symlink and not src.can_symlink:
+                            meta.symlink_error = "cannot symlink {}".format(src)
+                        if not meta.can_copy and not meta.can_symlink:
+                            meta.error = "neither copy or symlink supported: {}".format(
+                                meta.copy_error, meta.symlink_error
+                            )
+                    else:
+                        meta.error = "missing required file {}".format(src)
+                    if meta.error:
                         break
-                else:
-                    logging.debug("%s missing %s", cls.__name__, src)
-                    break
-                sources.append(src)
-            else:
-                return Meta(sources, can_copy, can_symlink)
+                    meta.sources.append(src)
+            return meta
         return None
 
+    @classmethod
+    def setup_meta(cls, interpreter):
+        return BuiltinViaGlobalRefMeta()
+
     @classmethod
     def sources(cls, interpreter):
         is_py2 = interpreter.version_info.major == 2

src/virtualenv/create/via_global_ref/store.py

@@ -0,0 +1,21 @@
+from virtualenv.util.path import Path
+
+
+def handle_store_python(meta, interpreter):
+    if is_store_python(interpreter):
+        meta.symlink_error = "Windows Store Python does not support virtual environments via symlink"
+    return meta
+
+
+def is_store_python(interpreter):
+    parts = Path(interpreter.system_executable).parts
+    return (
+        len(parts) > 4
+        and parts[-4] == "Microsoft"
+        and parts[-3] == "WindowsApps"
+        and parts[-2].startswith("PythonSoftwareFoundation.Python.3.")
+        and parts[-1].startswith("python")
+    )
+
+
+__all__ = ("handle_store_python", "is_store_python")

src/virtualenv/create/via_global_ref/venv.py

@@ -1,18 +1,15 @@
 from __future__ import absolute_import, unicode_literals
 
 import logging
-from collections import namedtuple
 from copy import copy
 
+from virtualenv.create.via_global_ref.store import handle_store_python
 from virtualenv.discovery.py_info import PythonInfo
 from virtualenv.error import ProcessCallFailed
-from virtualenv.info import fs_supports_symlink
 from virtualenv.util.path import ensure_dir
 from virtualenv.util.subprocess import run_cmd
 
-from .api import ViaGlobalRefApi
-
-Meta = namedtuple("Meta", ["can_symlink", "can_copy"])
+from .api import ViaGlobalRefApi, ViaGlobalRefMeta
 
 
 class Venv(ViaGlobalRefApi):
@@ -30,7 +27,10 @@ def _args(self):
     @classmethod
     def can_create(cls, interpreter):
         if interpreter.has_venv:
-            return Meta(can_symlink=fs_supports_symlink(), can_copy=True)
+            meta = ViaGlobalRefMeta()
+            if interpreter.platform == "win32" and interpreter.version_info.major == 3:
+                meta = handle_store_python(meta, interpreter)
+            return meta
         return None
 
     def create(self):

src/virtualenv/discovery/builtin.py

@@ -55,22 +55,29 @@ def get_interpreter(key, app_data=None):
 
 
 def propose_interpreters(spec, app_data):
-    # 1. if it's an absolute path and exists, use that
-    if spec.is_abs and os.path.exists(spec.path):
-        yield PythonInfo.from_exe(spec.path, app_data), True
-
-    # 2. try with the current
-    yield PythonInfo.current_system(app_data), True
-
-    # 3. otherwise fallback to platform default logic
-    if IS_WIN:
-        from .windows import propose_interpreters
+    # 1. if it's a path and exists
+    if spec.path is not None:
+        try:
+            os.lstat(spec.path)  # Windows Store Python does not work with os.path.exists, but does for os.lstat
+        except OSError:
+            if spec.is_abs:
+                raise
+        else:
+            yield PythonInfo.from_exe(os.path.abspath(spec.path), app_data), True
+        if spec.is_abs:
+            return
+    else:
+        # 2. otherwise try with the current
+        yield PythonInfo.current_system(app_data), True
 
-        for interpreter in propose_interpreters(spec, app_data):
-            yield interpreter, True
+        # 3. otherwise fallback to platform default logic
+        if IS_WIN:
+            from .windows import propose_interpreters
 
+            for interpreter in propose_interpreters(spec, app_data):
+                yield interpreter, True
+    # finally just find on path, the path order matters (as the candidates are less easy to control by end user)
     paths = get_paths()
-    # find on path, the path order matters (as the candidates are less easy to control by end user)
     tested_exes = set()
     for pos, path in enumerate(paths):
         path = ensure_text(path)

src/virtualenv/discovery/cached_py_info.py

@@ -65,11 +65,14 @@ def _get_via_file_cache(cls, py_info_cache, app_data, resolved_path, exe):
     key = sha256(str(resolved_path).encode("utf-8") if PY3 else str(resolved_path)).hexdigest()
     py_info = None
     resolved_path_text = ensure_text(str(resolved_path))
-    resolved_path_modified_timestamp = resolved_path.stat().st_mtime
+    try:
+        resolved_path_modified_timestamp = resolved_path.stat().st_mtime
+    except OSError:
+        resolved_path_modified_timestamp = -1
     data_file = py_info_cache / "{}.json".format(key)
     with py_info_cache.lock_for_key(key):
         data_file_path = data_file.path
-        if data_file_path.exists():  # if exists and matches load
+        if data_file_path.exists() and resolved_path_modified_timestamp != 1:  # if exists and matches load
             try:
                 data = json.loads(data_file_path.read_text())
                 if data["path"] == resolved_path_text and data["st_mtime"] == resolved_path_modified_timestamp:

src/virtualenv/discovery/py_info.py

@@ -419,7 +419,7 @@ def _find_possible_folders(self, inside_folder):
 
         # or at root level
         candidate_folder[inside_folder] = None
-        return list(candidate_folder.keys())
+        return list(i for i in candidate_folder.keys() if os.path.exists(i))
 
     def _find_possible_exe_names(self):
         name_candidate = OrderedDict()

src/virtualenv/discovery/py_spec.py

@@ -64,11 +64,13 @@ def _int_or_none(val):
                     arch = _int_or_none(groups["arch"])
 
             if not ok:
-                path = os.path.abspath(string_spec)
+                path = string_spec
 
         return cls(string_spec, impl, major, minor, micro, arch, path)
 
     def generate_names(self):
+        if self.implementation is None:
+            return
         impls = OrderedDict()
         if self.implementation:
             # first consider implementation as it is