Run CLI on Windows (#548)

appveyor.yml

@@ -0,0 +1,29 @@
+image: Visual Studio 2015
+
+environment:
+  # ensure Python 3.6 is first in path
+  PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%'
+  # GNU make
+  MAKE: C:\MinGW\bin\mingw32-make.exe
+  CLIENT_TEST_E2E_USER_NAME:
+    secure: eOtqZJ4yXsPlfNYTZ17U7NFym9KFIq1Cz/zb7dNLR4IAzz4lKtNRPKpGxqgS/BQZ8Kxbsf9EZ2EDKKC74P8kNHwb4oz++CJw6DLnzN+B3x4Nqg9gaMo/VB5yhlbxn90NPdhUl/m8j8pW0kWtwgah6U/Sh/7ZoIHycX2BgcbEONZjyRox2g9x1oKkFUEhVU+yU36EjEwrtu3Imft0yWLcXw==
+
+platform: x64
+
+install:
+  # install dev dependencies
+  - '%MAKE% -C python init'
+
+build: off
+
+# do not run automatic test discovery
+test: off
+
+test_script:
+  - '%MAKE% -C python test'
+  - '%MAKE% -C python _e2e_win'
+  - '%MAKE% -C python coverage'
+
+# will start RDP server you can connect to for remote debugging (a-la ssh in circleci)
+# on_finish:
+#   - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

python/Makefile

@@ -90,6 +90,16 @@ _e2e:
 		--verbose \
 		tests
 
+.PHONY: _e2e_win
+_e2e_win:
+	pytest \
+		-m "e2e and not no_win32" \
+		--cov=neuromation \
+		--cov-report term-missing:skip-covered \
+		--cov-report xml:coverage.xml \
+		--cov-append \
+		tests
+
 
 .PHONY: test
 test:
@@ -155,7 +165,8 @@ publish: dist publish-lint
 
 .PHONY: coverage
 coverage:
-	$(SHELL) <(curl -s https://codecov.io/bash) -X coveragepy
+	pip install codecov
+	codecov -f coverage.xml -X gcov
 
 .PHONY: format
 format:

python/neuromation/cli/command_handlers.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from pathlib import PosixPath, PurePosixPath
+from pathlib import Path
 from urllib.parse import ParseResult, urlparse
 
 
@@ -37,22 +37,22 @@ def _is_storage_path_url(self, path: ParseResult) -> None:
         if path.scheme != "storage":
             raise ValueError("Path should be targeting platform storage.")
 
-    def _render_platform_path(self, path_str: str) -> PosixPath:
-        target_path: PosixPath = PosixPath(path_str)
+    def _render_platform_path(self, path_str: str) -> Path:
+        target_path: Path = Path(path_str)
         if target_path.is_absolute():
-            target_path = target_path.relative_to(PosixPath("/"))
+            target_path = target_path.relative_to(Path("/"))
         return target_path
 
-    def _render_platform_path_with_principal(self, path: ParseResult) -> PurePosixPath:
-        target_path: PosixPath = self._render_platform_path(path.path)
+    def _render_platform_path_with_principal(self, path: ParseResult) -> Path:
+        target_path: Path = self._render_platform_path(path.path)
         target_principal = self._get_principal(path)
-        posix_path = PurePosixPath(PLATFORM_DELIMITER, target_principal, target_path)
+        posix_path = Path(PLATFORM_DELIMITER, target_principal, target_path)
         return posix_path
 
-    def render_uri_path_with_principal(self, path: str) -> PurePosixPath:
+    def render_uri_path_with_principal(self, path: str) -> Path:
         # Special case that shall be handled here, when path is '//'
         if path == "storage://":
-            return PosixPath(PLATFORM_DELIMITER)
+            return Path(PLATFORM_DELIMITER)
 
         # Normal processing flow
         path_url = urlparse(path, scheme="file")

python/neuromation/cli/const.py

@@ -1,3 +1,8 @@
+import sys
+
+
+WIN32 = sys.platform == "win32"
+
 # Python on Windows doesn't expose these constants in os module
 
 EX_CANTCREAT = 73

python/neuromation/cli/main.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import shutil
 import sys
@@ -18,6 +19,10 @@
 from .utils import Context, DeprecatedGroup, MainGroup, alias, format_example
 
 
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+
+
 log = logging.getLogger(__name__)
 
 

python/neuromation/cli/rc.py

@@ -14,6 +14,7 @@
 from neuromation.client.config import get_server_config
 from neuromation.client.users import get_token_username
 
+from .const import WIN32
 from .defaults import API_URL
 from .login import AuthConfig, AuthNegotiator, AuthToken
 
@@ -277,7 +278,7 @@ def _deserialize_auth_token(payload: Dict[str, Any]) -> Optional[AuthToken]:
 
 def _load(path: Path) -> Config:
     stat = path.stat()
-    if stat.st_mode & 0o777 != 0o600:
+    if not WIN32 and stat.st_mode & 0o777 != 0o600:
         raise RCException(
             f"Config file {path} has compromised permission bits, "
             f"run 'chmod 600 {path}' before usage"

python/neuromation/cli/storage.py

@@ -131,10 +131,11 @@ async def cp(
     dst = URL(destination)
 
     progress_obj = ProgressBase.create_progress(progress)
-    if not src.scheme:
-        src = URL(f"file:{src.path}")
-    if not dst.scheme:
-        dst = URL(f"file:{dst.path}")
+    # len(uri.scheme) == 1 is a workaround for Windows path like C:/path/to.txt
+    if not src.scheme or len(src.scheme) == 1:
+        src = URL(f"file:{source}")
+    if not dst.scheme or len(dst.scheme) == 1:
+        dst = URL(f"file:{destination}")
     async with cfg.make_client(timeout=timeout) as client:
         if src.scheme == "file" and dst.scheme == "storage":
             src = normalize_local_path_uri(src)

python/neuromation/client/storage.py

@@ -8,6 +8,7 @@
 from yarl import URL
 
 from neuromation.client.url_utils import (
+    _extract_path,
     normalize_local_path_uri,
     normalize_storage_path_uri,
 )
@@ -156,7 +157,7 @@ async def _iterate_file(
     async def upload_file(self, progress: AbstractProgress, src: URL, dst: URL) -> None:
         src = normalize_local_path_uri(src)
         dst = normalize_storage_path_uri(dst, self._config.username)
-        path = Path(src.path)
+        path = _extract_path(src)
         if not path.exists():
             raise FileNotFoundError(f"'{path}' does not exist")
         if path.is_dir():
@@ -188,7 +189,7 @@ async def upload_dir(self, progress: AbstractProgress, src: URL, dst: URL) -> No
             dst = dst / src.name
         src = normalize_local_path_uri(src)
         dst = normalize_storage_path_uri(dst, self._config.username)
-        path = Path(src.path).resolve()
+        path = _extract_path(src).resolve()
         if not path.exists():
             raise FileNotFoundError(f"{path} does not exist")
         if not path.is_dir():
@@ -215,7 +216,7 @@ async def download_file(
     ) -> None:
         src = normalize_storage_path_uri(src, self._config.username)
         dst = normalize_local_path_uri(dst)
-        path = Path(dst.path)
+        path = _extract_path(dst)
         if path.exists():
             if path.is_dir():
                 path = path / src.name
@@ -237,7 +238,7 @@ async def download_dir(
     ) -> None:
         src = normalize_storage_path_uri(src, self._config.username)
         dst = normalize_local_path_uri(dst)
-        path = Path(dst.path)
+        path = _extract_path(dst)
         path.mkdir(parents=True, exist_ok=True)
         for child in await self.ls(src):
             if child.is_file():

python/neuromation/client/url_utils.py

@@ -1,3 +1,5 @@
+import re
+import sys
 from pathlib import Path
 
 from yarl import URL
@@ -17,6 +19,9 @@ def normalize_storage_path_uri(uri: URL, username: str) -> URL:
         uri = URL("storage://" + username + "/" + uri.path)
     uri = uri.with_path(uri.path.lstrip("/"))
 
+    if "~" in uri.path:
+        raise ValueError(f"Cannot expand user for {uri}")
+
     return uri
 
 
@@ -29,8 +34,20 @@ def normalize_local_path_uri(uri: URL) -> URL:
         )
     if uri.host:
         raise ValueError(f"Host part is not allowed, found '{uri.host}'")
-    path = Path(uri.path).expanduser().absolute()
+    path = _extract_path(uri)
+    path = path.expanduser().absolute()
     ret = URL(path.as_uri())
+    if "~" in ret.path:
+        raise ValueError(f"Cannot expand user for {uri}")
     while ret.path.startswith("//"):
         ret = ret.with_path(ret.path[1:])
     return ret
+
+
+def _extract_path(uri: URL) -> Path:
+    path = Path(uri.path)
+    if sys.platform == "win32":
+        # result of previous normalization
+        if re.match(r"^[/\\][A-Za-z]:[/\\]", str(path)):
+            return Path(str(path)[1:])
+    return path

python/neuromation/utils.py

@@ -63,7 +63,7 @@ async def main():
                 warnings.simplefilter("ignore", ResourceWarning)
                 loop.close()
                 del loop
-                gc.collect(2)
+                gc.collect()
 
 
 def _cancel_all_tasks(

python/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts= --cov-branch
+addopts= --cov-branch --cov-report xml
 log_cli=false
 log_level=INFO
 filterwarnings=error

python/tests/cli/test_rc.py

@@ -1,4 +1,5 @@
 import logging
+import sys
 from pathlib import Path
 from textwrap import dedent
 from unittest import mock
@@ -334,6 +335,7 @@ def test_load_missing(nmrc):
     assert config == DEFAULTS
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="chmod 0o600 works on Posix only")
 def test_load_bad_file_mode(nmrc):
     document = """
         url: 'http://a.b/c'

python/tests/client/test_images.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import asynctest
 import pytest
@@ -564,6 +565,9 @@ async def message_generator():
 
 
 class TestRegistry:
+    @pytest.mark.skipif(
+        sys.platform == "win32", reason="aiodocker doens't support Windows pipes yet"
+    )
     async def test_ls(self, aiohttp_server, token):
         JSON = {"repositories": ["image://bob/alpine", "image://jill/bananas"]}