From ee17d4e6e23eee03668a711a92c54c9035121d43 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 22 Feb 2019 16:57:57 +0200 Subject: [PATCH 01/49] Run on windows --- python/neuromation/cli/const.py | 5 +++++ python/neuromation/cli/main.py | 15 ++++++++++++++- python/neuromation/cli/rc.py | 3 ++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/python/neuromation/cli/const.py b/python/neuromation/cli/const.py index 1a14bcf1d..50a51a331 100644 --- a/python/neuromation/cli/const.py +++ b/python/neuromation/cli/const.py @@ -1,3 +1,8 @@ +import sys + + +WIN32 = sys.platform == "win32" + # Python on Windows doesn't expose these constants in os module EX_CANTCREAT = 73 diff --git a/python/neuromation/cli/main.py b/python/neuromation/cli/main.py index f966e37b5..32a92d99a 100644 --- a/python/neuromation/cli/main.py +++ b/python/neuromation/cli/main.py @@ -1,3 +1,4 @@ +import asyncio import logging import shutil import sys @@ -13,11 +14,23 @@ from neuromation.cli.rc import RCException from . import completion, config, image, job, model, rc, share, storage -from .const import EX_DATAERR, EX_IOERR, EX_NOPERM, EX_OSFILE, EX_PROTOCOL, EX_SOFTWARE +from .const import ( + EX_DATAERR, + EX_IOERR, + EX_NOPERM, + EX_OSFILE, + EX_PROTOCOL, + EX_SOFTWARE, + WIN32, +) from .log_formatter import ConsoleHandler, ConsoleWarningFormatter from .utils import Context, DeprecatedGroup, MainGroup, alias, format_example +if WIN32: + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + log = logging.getLogger(__name__) diff --git a/python/neuromation/cli/rc.py b/python/neuromation/cli/rc.py index 9e15a2b2a..ee75f198f 100644 --- a/python/neuromation/cli/rc.py +++ b/python/neuromation/cli/rc.py @@ -15,6 +15,7 @@ from neuromation.client.users import get_token_username from neuromation.utils import run +from .const import WIN32 from .defaults import API_URL from .login import AuthConfig, AuthNegotiator, AuthToken @@ -281,7 +282,7 @@ def _deserialize_auth_token(payload: Dict[str, Any]) -> Optional[AuthToken]: def _load(path: Path) -> Config: stat = path.stat() - if stat.st_mode & 0o777 != 0o600: + if not WIN32 and stat.st_mode & 0o777 != 0o600: raise RCException( f"Config file {path} has compromised permission bits, " f"run 'chmod 600 {path}' before usage" From 2585a43e16166045ff92e2e3b599f0ff9c8dab9c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 22 Feb 2019 20:03:29 +0200 Subject: [PATCH 02/49] Reformat --- python/neuromation/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/neuromation/cli/utils.py b/python/neuromation/cli/utils.py index 06bb3dd80..ca3ebe047 100644 --- a/python/neuromation/cli/utils.py +++ b/python/neuromation/cli/utils.py @@ -1,7 +1,7 @@ import asyncio import re -import shlex from contextlib import suppress +import shlex from functools import wraps from typing import ( Any, From a888322a21946fd329d30e80d6a9a8ba20c3102d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 22 Feb 2019 20:03:53 +0200 Subject: [PATCH 03/49] Reformat --- python/neuromation/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/neuromation/cli/utils.py b/python/neuromation/cli/utils.py index ca3ebe047..06bb3dd80 100644 --- a/python/neuromation/cli/utils.py +++ b/python/neuromation/cli/utils.py @@ -1,7 +1,7 @@ import asyncio import re -from contextlib import suppress import shlex +from contextlib import suppress from functools import wraps from typing import ( Any, From cf4c80ed55e5eb5b1fd191b0ef4e3046f4f69e65 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 22 Feb 2019 20:25:19 +0200 Subject: [PATCH 04/49] Disable posix file permissions check --- python/tests/cli/test_rc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/tests/cli/test_rc.py b/python/tests/cli/test_rc.py index c47c5c7fd..562d01fd2 100644 --- a/python/tests/cli/test_rc.py +++ b/python/tests/cli/test_rc.py @@ -1,4 +1,5 @@ import logging +import sys from pathlib import Path from textwrap import dedent from unittest import mock @@ -334,6 +335,7 @@ def test_load_missing(nmrc): assert config == DEFAULTS +@pytest.mark.skipif(sys.platform == "win32", reason="chmod 0o600 works on Posix only") def test_load_bad_file_mode(nmrc): document = """ url: 'http://a.b/c' From c641a1a926ed0437170d0ecaae0d69a8051c714a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 11:35:58 +0200 Subject: [PATCH 05/49] Fix path extraction on Windows --- python/neuromation/client/storage.py | 9 +++++---- python/neuromation/client/url_utils.py | 14 +++++++++++++- python/tests/client/test_url_utils.py | 7 +++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/python/neuromation/client/storage.py b/python/neuromation/client/storage.py index 7beae8977..61c943dd2 100644 --- a/python/neuromation/client/storage.py +++ b/python/neuromation/client/storage.py @@ -10,6 +10,7 @@ from neuromation.client.url_utils import ( normalize_local_path_uri, normalize_storage_path_uri, + _extract_path, ) from .abc import AbstractProgress @@ -156,7 +157,7 @@ async def _iterate_file( async def upload_file(self, progress: AbstractProgress, src: URL, dst: URL) -> None: src = normalize_local_path_uri(src) dst = normalize_storage_path_uri(dst, self._config.username) - path = Path(src.path) + path = _extract_path(src) if not path.exists(): raise FileNotFoundError(f"'{path}' does not exist") if path.is_dir(): @@ -188,7 +189,7 @@ async def upload_dir(self, progress: AbstractProgress, src: URL, dst: URL) -> No dst = dst / src.name src = normalize_local_path_uri(src) dst = normalize_storage_path_uri(dst, self._config.username) - path = Path(src.path).resolve() + path = _extract_path(src).resolve() if not path.exists(): raise FileNotFoundError(f"{path} does not exist") if not path.is_dir(): @@ -215,7 +216,7 @@ async def download_file( ) -> None: src = normalize_storage_path_uri(src, self._config.username) dst = normalize_local_path_uri(dst) - path = Path(dst.path) + path = _extract_path(dst) if path.exists(): if path.is_dir(): path = path / src.name @@ -237,7 +238,7 @@ async def download_dir( ) -> None: src = normalize_storage_path_uri(src, self._config.username) dst = normalize_local_path_uri(dst) - path = Path(dst.path) + path = _extract_path(dst) path.mkdir(parents=True, exist_ok=True) for child in await self.ls(src): if child.is_file(): diff --git a/python/neuromation/client/url_utils.py b/python/neuromation/client/url_utils.py index d47ac0c92..5efd2b916 100644 --- a/python/neuromation/client/url_utils.py +++ b/python/neuromation/client/url_utils.py @@ -1,3 +1,5 @@ +import sys +import re from pathlib import Path from yarl import URL @@ -29,8 +31,18 @@ def normalize_local_path_uri(uri: URL) -> URL: ) if uri.host: raise ValueError(f"Host part is not allowed, found '{uri.host}'") - path = Path(uri.path).expanduser().absolute() + path = _extract_path(uri) + path = path.expanduser().absolute() ret = URL(path.as_uri()) while ret.path.startswith("//"): ret = ret.with_path(ret.path[1:]) return ret + + +def _extract_path(uri: URL) -> Path: + path = Path(uri.path) + if sys.platform == 'win32': + # result of previous normalization + if re.match(r'[/\\][A-Za-z]:[/\\]', str(path)): + return Path(str(path)[1:]) + return path diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index 8c739a2ca..2155b11cd 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -1,4 +1,5 @@ from pathlib import Path +import sys import pytest from yarl import URL @@ -242,3 +243,9 @@ async def test_normalize_local_path_uri__3_slashes__double(token, fake_homedir): assert url.host is None assert url.path == "/path/to/file.txt" assert str(url) == "file:///path/to/file.txt" + + +@pytest.mark.skipif(sys.platform != 'win32', reason='Requires Windows') +def test_normalized_path(): + p = URL('file:///Z:/neuromation/platform-api-clients/python/setup.py') + assert normalize_local_path_uri(p) == p From 417f5be62fb65a6d1094cfff4ebfb6f8824543ae Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 11:36:16 +0200 Subject: [PATCH 06/49] Reformat --- python/neuromation/client/storage.py | 2 +- python/neuromation/client/url_utils.py | 6 +++--- python/tests/client/test_url_utils.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/neuromation/client/storage.py b/python/neuromation/client/storage.py index 61c943dd2..ec3209efa 100644 --- a/python/neuromation/client/storage.py +++ b/python/neuromation/client/storage.py @@ -8,9 +8,9 @@ from yarl import URL from neuromation.client.url_utils import ( + _extract_path, normalize_local_path_uri, normalize_storage_path_uri, - _extract_path, ) from .abc import AbstractProgress diff --git a/python/neuromation/client/url_utils.py b/python/neuromation/client/url_utils.py index 5efd2b916..0657e0829 100644 --- a/python/neuromation/client/url_utils.py +++ b/python/neuromation/client/url_utils.py @@ -1,5 +1,5 @@ -import sys import re +import sys from pathlib import Path from yarl import URL @@ -41,8 +41,8 @@ def normalize_local_path_uri(uri: URL) -> URL: def _extract_path(uri: URL) -> Path: path = Path(uri.path) - if sys.platform == 'win32': + if sys.platform == "win32": # result of previous normalization - if re.match(r'[/\\][A-Za-z]:[/\\]', str(path)): + if re.match(r"[/\\][A-Za-z]:[/\\]", str(path)): return Path(str(path)[1:]) return path diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index 2155b11cd..481873d72 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -1,5 +1,5 @@ -from pathlib import Path import sys +from pathlib import Path import pytest from yarl import URL @@ -245,7 +245,7 @@ async def test_normalize_local_path_uri__3_slashes__double(token, fake_homedir): assert str(url) == "file:///path/to/file.txt" -@pytest.mark.skipif(sys.platform != 'win32', reason='Requires Windows') +@pytest.mark.skipif(sys.platform != "win32", reason="Requires Windows") def test_normalized_path(): - p = URL('file:///Z:/neuromation/platform-api-clients/python/setup.py') + p = URL("file:///Z:/neuromation/platform-api-clients/python/setup.py") assert normalize_local_path_uri(p) == p From 1a5dacccdd5ad9c3261c5797200a170b90997a0b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 11:37:40 +0200 Subject: [PATCH 07/49] Correct regexp --- python/neuromation/client/url_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/neuromation/client/url_utils.py b/python/neuromation/client/url_utils.py index 0657e0829..81b01fab5 100644 --- a/python/neuromation/client/url_utils.py +++ b/python/neuromation/client/url_utils.py @@ -43,6 +43,6 @@ def _extract_path(uri: URL) -> Path: path = Path(uri.path) if sys.platform == "win32": # result of previous normalization - if re.match(r"[/\\][A-Za-z]:[/\\]", str(path)): + if re.match(r"^[/\\][A-Za-z]:[/\\]", str(path)): return Path(str(path)[1:]) return path From ab171bc428cb4e9da37e1a9d739642d38de8f134 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 11:50:20 +0200 Subject: [PATCH 08/49] Work on --- python/tests/client/test_url_utils.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index 481873d72..7010a134f 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -6,6 +6,7 @@ from neuromation.client import Client from neuromation.client.url_utils import ( + _extract_path, normalize_local_path_uri, normalize_storage_path_uri, ) @@ -134,8 +135,8 @@ async def test_normalize_local_path_uri__tilde_in_relative_path(token, fake_home url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None - assert url.path == f"{fake_homedir}/path/to/file.txt" - assert str(url) == f"file://{fake_homedir}/path/to/file.txt" + assert _extract_path(url) == fake_homedir / "path/to/file.txt" + assert str(url) == (fake_homedir / "path/to/file.txt").as_uri() async def test_normalize_storage_path_uri__tilde_in_absolute_path(token, client): @@ -221,8 +222,8 @@ async def test_normalize_local_path_uri__tilde_slash__double(token, fake_homedir url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None - assert url.path == f"{fake_homedir}/path/to/file.txt" - assert str(url) == f"file://{fake_homedir}/path/to/file.txt" + assert _extract_path(url) == fake_homedir / "path/to/file.txt" + assert str(url) == (fake_homedir / "path/to/file.txt").as_uri() async def test_normalize_storage_path_uri__3_slashes__double(token, client): @@ -235,14 +236,14 @@ async def test_normalize_storage_path_uri__3_slashes__double(token, client): assert str(url) == "storage://user/path/to/file.txt" -async def test_normalize_local_path_uri__3_slashes__double(token, fake_homedir): - url = URL("file:///path/to/file.txt") +async def test_normalize_local_path_uri__3_slashes__double(token, pwd): + url = URL(f"file:///{pwd}/path/to/file.txt") url = normalize_local_path_uri(url) url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None - assert url.path == "/path/to/file.txt" - assert str(url) == "file:///path/to/file.txt" + assert _extract_path(url) == pwd / "path/to/file.txt" + assert str(url) == (pwd / "path/to/file.txt").as_uri() @pytest.mark.skipif(sys.platform != "win32", reason="Requires Windows") From 9bab16df99cdebf0ca50f382ec98ac12e5110aad Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 12:11:37 +0200 Subject: [PATCH 09/49] Forbid tilda in path after normalization --- python/neuromation/client/url_utils.py | 5 ++++ python/tests/client/test_url_utils.py | 39 +++++++------------------- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/python/neuromation/client/url_utils.py b/python/neuromation/client/url_utils.py index 81b01fab5..974d6ea05 100644 --- a/python/neuromation/client/url_utils.py +++ b/python/neuromation/client/url_utils.py @@ -19,6 +19,9 @@ def normalize_storage_path_uri(uri: URL, username: str) -> URL: uri = URL("storage://" + username + "/" + uri.path) uri = uri.with_path(uri.path.lstrip("/")) + if '~' in uri.path: + raise ValueError(f"Cannot expand user for {uri}") + return uri @@ -34,6 +37,8 @@ def normalize_local_path_uri(uri: URL) -> URL: path = _extract_path(uri) path = path.expanduser().absolute() ret = URL(path.as_uri()) + if '~' in ret.path: + raise ValueError(f"Cannot expand user for {uri}") while ret.path.startswith("//"): ret = ret.with_path(ret.path[1:]) return ret diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index 7010a134f..e1670e3da 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -123,11 +123,8 @@ async def test_normalize_local_path_uri__4_slashes_relative(token, pwd): async def test_normalize_storage_path_uri__tilde_in_relative_path(token, client): url = URL("storage:~/path/to/file.txt") - url = normalize_storage_path_uri(url, client.username) - assert url.scheme == "storage" - assert url.host == "user" - assert url.path == "/~/path/to/file.txt" - assert str(url) == "storage://user/~/path/to/file.txt" + with pytest.raises(ValueError, match=".*Cannot expand user.*"): + normalize_storage_path_uri(url, client.username) async def test_normalize_local_path_uri__tilde_in_relative_path(token, fake_homedir): @@ -141,20 +138,14 @@ async def test_normalize_local_path_uri__tilde_in_relative_path(token, fake_home async def test_normalize_storage_path_uri__tilde_in_absolute_path(token, client): url = URL("storage:/~/path/to/file.txt") - url = normalize_storage_path_uri(url, client.username) - assert url.scheme == "storage" - assert url.host == "user" - assert url.path == "/~/path/to/file.txt" - assert str(url) == "storage://user/~/path/to/file.txt" + with pytest.raises(ValueError, match=".*Cannot expand user.*"): + normalize_storage_path_uri(url, client.username) -async def test_normalize_local_path_uri__tilde_in_absolute_path(token): +async def test_normalize_local_path_uri__tilde_in_absolute_path(token, fake_homedir): url = URL("file:/~/path/to/file.txt") - url = normalize_local_path_uri(url) - assert url.scheme == "file" - assert url.host is None - assert url.path == "/~/path/to/file.txt" - assert str(url) == "file:///~/path/to/file.txt" + with pytest.raises(ValueError, match=".*Cannot expand user.*"): + normalize_local_path_uri(url) async def test_normalize_storage_path_uri__tilde_in_host(token, client): @@ -189,7 +180,6 @@ async def test_normalize_local_path_uri__bad_scheme(token): async def test_normalize_storage_path_uri__no_slash__double(token, client): url = URL("storage:path/to/file.txt") url = normalize_storage_path_uri(url, client.username) - url = normalize_storage_path_uri(url, client.username) assert url.scheme == "storage" assert url.host == "user" assert url.path == "/path/to/file.txt" @@ -199,27 +189,20 @@ async def test_normalize_storage_path_uri__no_slash__double(token, client): async def test_normalize_local_path_uri__no_slash__double(token, pwd): url = URL("file:path/to/file.txt") url = normalize_local_path_uri(url) - url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None - assert url.path == f"{pwd}/path/to/file.txt" - assert str(url) == f"file://{pwd}/path/to/file.txt" + assert _extract_path(url) == pwd / "path/to/file.txt" async def test_normalize_storage_path_uri__tilde_slash__double(token, client): url = URL("storage:~/path/to/file.txt") - url = normalize_storage_path_uri(url, client.username) - url = normalize_storage_path_uri(url, client.username) - assert url.scheme == "storage" - assert url.host == "user" - assert url.path == "/~/path/to/file.txt" - assert str(url) == "storage://user/~/path/to/file.txt" + with pytest.raises(ValueError, match=".*Cannot expand user.*"): + normalize_storage_path_uri(url, client.username) async def test_normalize_local_path_uri__tilde_slash__double(token, fake_homedir): url = URL("file:~/path/to/file.txt") url = normalize_local_path_uri(url) - url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None assert _extract_path(url) == fake_homedir / "path/to/file.txt" @@ -229,7 +212,6 @@ async def test_normalize_local_path_uri__tilde_slash__double(token, fake_homedir async def test_normalize_storage_path_uri__3_slashes__double(token, client): url = URL("storage:///path/to/file.txt") url = normalize_storage_path_uri(url, client.username) - url = normalize_storage_path_uri(url, client.username) assert url.scheme == "storage" assert url.host == "user" assert url.path == "/path/to/file.txt" @@ -239,7 +221,6 @@ async def test_normalize_storage_path_uri__3_slashes__double(token, client): async def test_normalize_local_path_uri__3_slashes__double(token, pwd): url = URL(f"file:///{pwd}/path/to/file.txt") url = normalize_local_path_uri(url) - url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None assert _extract_path(url) == pwd / "path/to/file.txt" From 641e96cb3166800cc45dcb3eb4c79c970d354100 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 12:12:43 +0200 Subject: [PATCH 10/49] Reformat --- python/neuromation/client/url_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/neuromation/client/url_utils.py b/python/neuromation/client/url_utils.py index 974d6ea05..02f404045 100644 --- a/python/neuromation/client/url_utils.py +++ b/python/neuromation/client/url_utils.py @@ -19,7 +19,7 @@ def normalize_storage_path_uri(uri: URL, username: str) -> URL: uri = URL("storage://" + username + "/" + uri.path) uri = uri.with_path(uri.path.lstrip("/")) - if '~' in uri.path: + if "~" in uri.path: raise ValueError(f"Cannot expand user for {uri}") return uri @@ -37,7 +37,7 @@ def normalize_local_path_uri(uri: URL) -> URL: path = _extract_path(uri) path = path.expanduser().absolute() ret = URL(path.as_uri()) - if '~' in ret.path: + if "~" in ret.path: raise ValueError(f"Cannot expand user for {uri}") while ret.path.startswith("//"): ret = ret.with_path(ret.path[1:]) From fe99164bd488f318f71bad674b1a41999a674805 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 12:53:06 +0200 Subject: [PATCH 11/49] Pass client tests on windows --- python/tests/client/test_images.py | 3 +++ python/tests/client/test_url_utils.py | 14 ++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/python/tests/client/test_images.py b/python/tests/client/test_images.py index a30afc66e..5cde5851e 100644 --- a/python/tests/client/test_images.py +++ b/python/tests/client/test_images.py @@ -262,6 +262,9 @@ async def message_generator(): class TestRegistry: + @pytest.mark.skipif( + sys.platform == "win32", reason="aiodocker doens't support Windows pipes yet" + ) async def test_ls(self, aiohttp_server, token): JSON = {"repositories": ["image://bob/alpine", "image://jill/bananas"]} diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index e1670e3da..7b25908e5 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -48,8 +48,7 @@ async def test_normalize_local_path_uri__0_slashes_relative(token, pwd): url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None - assert url.path == f"{pwd}/path/to/file.txt" - assert str(url) == f"file://{pwd}/path/to/file.txt" + assert _extract_path(url) == pwd / "path/to/file.txt" async def test_normalize_storage_path_uri__1_slash_absolute(token, client): @@ -66,8 +65,7 @@ async def test_normalize_local_path_uri__1_slash_absolute(token, pwd): url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None - assert url.path == "/path/to/file.txt" - assert str(url) == "file:///path/to/file.txt" + assert _extract_path(url) == Path(pwd.drive + "/path/to/file.txt") async def test_normalize_storage_path_uri__2_slashes(token, client): @@ -99,8 +97,7 @@ async def test_normalize_local_path_uri__3_slashes_relative(token, pwd): url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None - assert url.path == "/path/to/file.txt" - assert str(url) == "file:///path/to/file.txt" + assert _extract_path(url) == Path(pwd.drive + "/path/to/file.txt") async def test_normalize_storage_path_uri__4_slashes_relative(token, client): @@ -112,13 +109,14 @@ async def test_normalize_storage_path_uri__4_slashes_relative(token, client): assert str(url) == "storage://user/path/to/file.txt" -async def test_normalize_local_path_uri__4_slashes_relative(token, pwd): +@pytest.mark.skipif(sys.platform != "posix", reason="Doesn't work on Windows") +async def test_normalize_local_path_uri__4_slashes_relative(): url = URL("file:////path/to/file.txt") url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None assert url.path == "/path/to/file.txt" - assert str(url) == "file:///path/to/file.txt" + assert str(url) == f"file:///{dirve}path/to/file.txt" async def test_normalize_storage_path_uri__tilde_in_relative_path(token, client): From 6b9709eefd23b6749e9eddea2db18645b7dafd24 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 12:53:26 +0200 Subject: [PATCH 12/49] Reformat --- python/tests/client/test_images.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tests/client/test_images.py b/python/tests/client/test_images.py index 5cde5851e..6c6fdfd05 100644 --- a/python/tests/client/test_images.py +++ b/python/tests/client/test_images.py @@ -1,4 +1,5 @@ import os +import sys import asynctest import pytest From 3cda29e0b7c38e5fab3821b1a23412a59264fc80 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 12:53:59 +0200 Subject: [PATCH 13/49] Fix typo --- python/tests/client/test_url_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index 7b25908e5..1d70c6f8f 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -109,14 +109,14 @@ async def test_normalize_storage_path_uri__4_slashes_relative(token, client): assert str(url) == "storage://user/path/to/file.txt" -@pytest.mark.skipif(sys.platform != "posix", reason="Doesn't work on Windows") +@pytest.mark.skipif(sys.platform != 'posix', reason="Doesn't work on Windows") async def test_normalize_local_path_uri__4_slashes_relative(): url = URL("file:////path/to/file.txt") url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None assert url.path == "/path/to/file.txt" - assert str(url) == f"file:///{dirve}path/to/file.txt" + assert str(url) == f"file:///{drive}path/to/file.txt" async def test_normalize_storage_path_uri__tilde_in_relative_path(token, client): From 277234c97f3f8b71035800e66cf9c4dfb788ace8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 12:54:26 +0200 Subject: [PATCH 14/49] Reformat --- python/tests/client/test_url_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index 1d70c6f8f..5cf02f72e 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -109,7 +109,7 @@ async def test_normalize_storage_path_uri__4_slashes_relative(token, client): assert str(url) == "storage://user/path/to/file.txt" -@pytest.mark.skipif(sys.platform != 'posix', reason="Doesn't work on Windows") +@pytest.mark.skipif(sys.platform != "posix", reason="Doesn't work on Windows") async def test_normalize_local_path_uri__4_slashes_relative(): url = URL("file:////path/to/file.txt") url = normalize_local_path_uri(url) From 83aa73b0fcb3333e3375025b0aff5feb34f092f8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 12:55:17 +0200 Subject: [PATCH 15/49] Fix test --- python/tests/client/test_url_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/client/test_url_utils.py b/python/tests/client/test_url_utils.py index 5cf02f72e..b99f2e0a5 100644 --- a/python/tests/client/test_url_utils.py +++ b/python/tests/client/test_url_utils.py @@ -109,14 +109,14 @@ async def test_normalize_storage_path_uri__4_slashes_relative(token, client): assert str(url) == "storage://user/path/to/file.txt" -@pytest.mark.skipif(sys.platform != "posix", reason="Doesn't work on Windows") +@pytest.mark.skipif(sys.platform == "win32", reason="Doesn't work on Windows") async def test_normalize_local_path_uri__4_slashes_relative(): url = URL("file:////path/to/file.txt") url = normalize_local_path_uri(url) assert url.scheme == "file" assert url.host is None assert url.path == "/path/to/file.txt" - assert str(url) == f"file:///{drive}path/to/file.txt" + assert str(url) == f"file:///path/to/file.txt" async def test_normalize_storage_path_uri__tilde_in_relative_path(token, client): From 472fb148d9e8619d710b23a2ff560682d08242e0 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 13:32:11 +0200 Subject: [PATCH 16/49] Add appveyor --- appveyor.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..0e9211472 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,23 @@ +image: Visual Studio 2015 + +environment: + # ensure Python 3.6 is first in path + PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' + # GNU make + MAKE: C:\MinGW\bin\mingw32-make.exe + +platform: x64 + +install: + # install dev dependencies + - '%MAKE% -C python init' + +# do not run automatic test discovery +test: off + +build_script: + - '%MAKE% -C python test' + +# will start RDP server you can connect to for remote debugging (a-la ssh in circleci) +on_finish: + - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) From f3ddacb7e24cafdc7ffc232af79d36d700ffc8b9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 13:56:41 +0200 Subject: [PATCH 17/49] Make pypi work --- python/neuromation/cli/main.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/python/neuromation/cli/main.py b/python/neuromation/cli/main.py index 32a92d99a..a03104716 100644 --- a/python/neuromation/cli/main.py +++ b/python/neuromation/cli/main.py @@ -14,20 +14,12 @@ from neuromation.cli.rc import RCException from . import completion, config, image, job, model, rc, share, storage -from .const import ( - EX_DATAERR, - EX_IOERR, - EX_NOPERM, - EX_OSFILE, - EX_PROTOCOL, - EX_SOFTWARE, - WIN32, -) +from .const import EX_DATAERR, EX_IOERR, EX_NOPERM, EX_OSFILE, EX_PROTOCOL, EX_SOFTWARE from .log_formatter import ConsoleHandler, ConsoleWarningFormatter from .utils import Context, DeprecatedGroup, MainGroup, alias, format_example -if WIN32: +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) From acfa57d6191f1594289da18675e58df0151d35e1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 14:06:34 +0200 Subject: [PATCH 18/49] Disable RDP --- appveyor.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0e9211472..041deb8bf 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,8 +16,11 @@ install: test: off build_script: - - '%MAKE% -C python test' + - pip3 install -r requirements/ci.txt + +test_script: + - '%MAKE% -C python ci' # will start RDP server you can connect to for remote debugging (a-la ssh in circleci) -on_finish: - - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) +# on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) From 732db7e11c95bd36e6e5a06dbeb82b65af7b54c3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 14:54:54 +0200 Subject: [PATCH 19/49] Use a small timeout for disabline ugly message about closed loop --- python/neuromation/cli/utils.py | 2 ++ python/neuromation/utils.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/neuromation/cli/utils.py b/python/neuromation/cli/utils.py index 06bb3dd80..4ee20cd8a 100644 --- a/python/neuromation/cli/utils.py +++ b/python/neuromation/cli/utils.py @@ -43,6 +43,8 @@ async def _run_async_function( await task await version_checker.close() + await asyncio.sleep(0.05) + def run_async(callback: Callable[..., Awaitable[_T]]) -> Callable[..., _T]: @wraps(callback) diff --git a/python/neuromation/utils.py b/python/neuromation/utils.py index 8043df3b6..1d8bc5e84 100644 --- a/python/neuromation/utils.py +++ b/python/neuromation/utils.py @@ -63,7 +63,7 @@ async def main(): warnings.simplefilter("ignore", ResourceWarning) loop.close() del loop - gc.collect(2) + gc.collect() def _cancel_all_tasks( From 47b75203e66e7f7e85615d78e4f4d29518b5d209 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 15:03:05 +0200 Subject: [PATCH 20/49] Explicitly accept config in run_async decorator --- python/neuromation/cli/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/neuromation/cli/utils.py b/python/neuromation/cli/utils.py index 4ee20cd8a..b9ae5fe59 100644 --- a/python/neuromation/cli/utils.py +++ b/python/neuromation/cli/utils.py @@ -21,6 +21,7 @@ from neuromation.client import Volume from neuromation.utils import run +from .rc import Config from .version_utils import VersionChecker @@ -30,13 +31,13 @@ async def _run_async_function( - func: Callable[..., Awaitable[_T]], *args: Any, **kwargs: Any + func: Callable[..., Awaitable[_T]], cfg: Config, *args: Any, **kwargs: Any ) -> _T: loop = asyncio.get_event_loop() version_checker = VersionChecker() task = loop.create_task(version_checker.run()) try: - return await func(*args, **kwargs) + return await func(cfg, *args, **kwargs) finally: task.cancel() with suppress(asyncio.CancelledError): @@ -48,8 +49,8 @@ async def _run_async_function( def run_async(callback: Callable[..., Awaitable[_T]]) -> Callable[..., _T]: @wraps(callback) - def wrapper(*args: Any, **kwargs: Any) -> _T: - return run(_run_async_function(callback, *args, **kwargs)) + def wrapper(cfg: Config, *args: Any, **kwargs: Any) -> _T: + return run(_run_async_function(callback, cfg, *args, **kwargs)) return wrapper From e562743be468878d303f917a7283d5a753d387da Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 15:55:19 +0200 Subject: [PATCH 21/49] Fix path --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 041deb8bf..71ac20adb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,7 +16,7 @@ install: test: off build_script: - - pip3 install -r requirements/ci.txt + - pip3 install -r python/requirements/ci.txt test_script: - '%MAKE% -C python ci' From bde12d477afa4a4a3cded0524da7292d7c1c8e4d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:02:11 +0200 Subject: [PATCH 22/49] Change appveyor config --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 71ac20adb..23dc14fd1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,7 +16,7 @@ install: test: off build_script: - - pip3 install -r python/requirements/ci.txt + - cd python && pip3 install -r requirements/ci.txt test_script: - '%MAKE% -C python ci' From e7af5bb495653d3d927108a1919a4b12da14d76b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:09:34 +0200 Subject: [PATCH 23/49] Replace cd with pushd / popd --- appveyor.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 23dc14fd1..6b02d49cc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,7 +16,9 @@ install: test: off build_script: - - cd python && pip3 install -r requirements/ci.txt + - pushd python + - pip3 install -r requirements/ci.txt + - popd test_script: - '%MAKE% -C python ci' From e6551691dc875c67b89518c4f972cc4c8eb84d16 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:12:30 +0200 Subject: [PATCH 24/49] Tune build --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 6b02d49cc..2e3ec0784 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,9 +16,10 @@ install: test: off build_script: + - pip install -U pip - pushd python - pip3 install -r requirements/ci.txt - - popd + - popd || true test_script: - '%MAKE% -C python ci' From 971feb2079b0e539b8e84fd46632cd3bf9a04aa3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:19:15 +0200 Subject: [PATCH 25/49] Use pip --user --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2e3ec0784..bd064f7c5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,9 +16,9 @@ install: test: off build_script: - - pip install -U pip + - pip install --user -U pip - pushd python - - pip3 install -r requirements/ci.txt + - pip3 install --user -r requirements/ci.txt - popd || true test_script: From 0031f5ee66a4457ea281a05c74c2e4c605c3d343 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:20:36 +0200 Subject: [PATCH 26/49] Don't install twice --- appveyor.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index bd064f7c5..b84e8e63f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,12 +15,6 @@ install: # do not run automatic test discovery test: off -build_script: - - pip install --user -U pip - - pushd python - - pip3 install --user -r requirements/ci.txt - - popd || true - test_script: - '%MAKE% -C python ci' From b94f4d6b98a40717a11e8ea462e2682b7425d36f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:25:43 +0200 Subject: [PATCH 27/49] Build: off --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index b84e8e63f..068cb203c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,6 +12,8 @@ install: # install dev dependencies - '%MAKE% -C python init' +build: off + # do not run automatic test discovery test: off From 013f5a88fafed7b8fe0f059466908ce92e536a33 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:32:27 +0200 Subject: [PATCH 28/49] Work on windows --- python/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Makefile b/python/Makefile index c8f154a81..d5194660a 100644 --- a/python/Makefile +++ b/python/Makefile @@ -196,7 +196,7 @@ docs: .PHONY: lint-docs lint-docs: TMP:=$(shell mktemp $${TMPDIR:-/tmp}/README.XXXXXXXX.md) lint-docs: - build-tools/cli-help-generator.py README.in.md ${TMP} + python build-tools/cli-help-generator.py README.in.md ${TMP} markdown-toc -t github -h 6 ${TMP} diff -q ${TMP} README.md From 91a43ca42c087db6ae79eaf5eb395cb14ea00c93 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:39:54 +0200 Subject: [PATCH 29/49] Tune test steps --- appveyor.yml | 4 +++- python/Makefile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 068cb203c..434360df7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,7 +18,9 @@ build: off test: off test_script: - - '%MAKE% -C python ci' + - '%MAKE% -C python test' + - '%MAKE% -C python _e2e' + - '%MAKE% -C python coverage' # will start RDP server you can connect to for remote debugging (a-la ssh in circleci) # on_finish: diff --git a/python/Makefile b/python/Makefile index d5194660a..c8f154a81 100644 --- a/python/Makefile +++ b/python/Makefile @@ -196,7 +196,7 @@ docs: .PHONY: lint-docs lint-docs: TMP:=$(shell mktemp $${TMPDIR:-/tmp}/README.XXXXXXXX.md) lint-docs: - python build-tools/cli-help-generator.py README.in.md ${TMP} + build-tools/cli-help-generator.py README.in.md ${TMP} markdown-toc -t github -h 6 ${TMP} diff -q ${TMP} README.md From b712b338b1b09e6a7974942d68bab1531d9a9e20 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:43:14 +0200 Subject: [PATCH 30/49] Don't use fork on windows --- appveyor.yml | 2 +- python/Makefile | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 434360df7..e9efad62d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ test: off test_script: - '%MAKE% -C python test' - - '%MAKE% -C python _e2e' + - '%MAKE% -C python _e2e_win' - '%MAKE% -C python coverage' # will start RDP server you can connect to for remote debugging (a-la ssh in circleci) diff --git a/python/Makefile b/python/Makefile index c8f154a81..6d7283329 100644 --- a/python/Makefile +++ b/python/Makefile @@ -88,6 +88,16 @@ _e2e: --cov-append \ tests +.PHONY: _e2e_win +_e2e_win: + pytest \ + -m "e2e" \ + --cov=neuromation \ + --cov-report term-missing:skip-covered \ + --cov-report xml:coverage.xml \ + --cov-append \ + tests + .PHONY: test test: From c440660033336ed1ac9c8a35a91b9260f550ad2f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 16:56:58 +0200 Subject: [PATCH 31/49] Use secure CLIENT_TEST_E2E_USER_NAME env --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index e9efad62d..39a7175c1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,6 +5,8 @@ environment: PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' # GNU make MAKE: C:\MinGW\bin\mingw32-make.exe + CLIENT_TEST_E2E_USER_NAME: + secure: eOtqZJ4yXsPlfNYTZ17U7NFym9KFIq1Cz/zb7dNLR4IAzz4lKtNRPKpGxqgS/BQZ8Kxbsf9EZ2EDKKC74P8kNHwb4oz++CJw6DLnzN+B3x4Nqg9gaMo/VB5yhlbxn90NPdhUl/m8j8pW0kWtwgah6U/Sh/7ZoIHycX2BgcbEONZjyRox2g9x1oKkFUEhVU+yU36EjEwrtu3Imft0yWLcXw== platform: x64 From f0c2452c058e178bc3acc691f5cbaa78a0fb7d91 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 17:21:53 +0200 Subject: [PATCH 32/49] Enable proactor on windows tests --- python/tests/e2e/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index dfcad5c5e..27f9a1bc9 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -10,6 +10,7 @@ from pathlib import Path from time import sleep, time from uuid import uuid4 as uuid +import sys import pytest from yarl import URL @@ -37,6 +38,10 @@ ) +if sys.platform == 'win32': + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + class TestRetriesExceeded(Exception): pass From 99a5d69167b7e8bd05f0a308637c2aa7377e709d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 17:26:25 +0200 Subject: [PATCH 33/49] Reformat --- python/tests/e2e/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index 27f9a1bc9..2d598a04b 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -3,6 +3,7 @@ import logging import os import re +import sys from collections import namedtuple from contextlib import suppress from hashlib import sha1 @@ -10,7 +11,6 @@ from pathlib import Path from time import sleep, time from uuid import uuid4 as uuid -import sys import pytest from yarl import URL @@ -38,7 +38,7 @@ ) -if sys.platform == 'win32': +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) From d4b952c7414df7bd41080f59e1f2e835b72483f1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 19:02:01 +0200 Subject: [PATCH 34/49] Work on --- python/neuromation/cli/storage.py | 5 +++-- python/tests/e2e/conftest.py | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/python/neuromation/cli/storage.py b/python/neuromation/cli/storage.py index 0bb1ddf5f..f489e66e0 100644 --- a/python/neuromation/cli/storage.py +++ b/python/neuromation/cli/storage.py @@ -134,9 +134,10 @@ async def cp( dst = URL(destination) progress_obj = ProgressBase.create_progress(progress) - if not src.scheme: + # len(uri.scheme) == 1 is a workaround for Windows path like C:/path/to.txt + if not src.scheme or len(src.scheme) == 1: src = URL(f"file:{src.path}") - if not dst.scheme: + if not dst.scheme or len(dst.scheme) == 1: dst = URL(f"file:{dst.path}") async with cfg.make_client(timeout=timeout) as client: if src.scheme == "file" and dst.scheme == "storage": diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index 2d598a04b..6c877e164 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -38,10 +38,6 @@ ) -if sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) - - class TestRetriesExceeded(Exception): pass @@ -51,6 +47,8 @@ class TestRetriesExceeded(Exception): def run_async(coro): def wrapper(*args, **kwargs): + if sys.platform == 'win32': + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) return run(coro(*args, **kwargs)) return wrapper From a9ad63ba6d074903de2a6d13fb0a437c8b80cf4f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 19:21:10 +0200 Subject: [PATCH 35/49] Get rid of PosixPath --- python/neuromation/cli/command_handlers.py | 18 +++++++++--------- python/tests/e2e/conftest.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/python/neuromation/cli/command_handlers.py b/python/neuromation/cli/command_handlers.py index 15c58a881..0113ca401 100644 --- a/python/neuromation/cli/command_handlers.py +++ b/python/neuromation/cli/command_handlers.py @@ -1,6 +1,6 @@ import logging import os -from pathlib import PosixPath, PurePosixPath +from pathlib import Path from urllib.parse import ParseResult, urlparse @@ -37,22 +37,22 @@ def _is_storage_path_url(self, path: ParseResult) -> None: if path.scheme != "storage": raise ValueError("Path should be targeting platform storage.") - def _render_platform_path(self, path_str: str) -> PosixPath: - target_path: PosixPath = PosixPath(path_str) + def _render_platform_path(self, path_str: str) -> Path: + target_path: Path = Path(path_str) if target_path.is_absolute(): - target_path = target_path.relative_to(PosixPath("/")) + target_path = target_path.relative_to(Path("/")) return target_path - def _render_platform_path_with_principal(self, path: ParseResult) -> PurePosixPath: - target_path: PosixPath = self._render_platform_path(path.path) + def _render_platform_path_with_principal(self, path: ParseResult) -> Path: + target_path: Path = self._render_platform_path(path.path) target_principal = self._get_principal(path) - posix_path = PurePosixPath(PLATFORM_DELIMITER, target_principal, target_path) + posix_path = Path(PLATFORM_DELIMITER, target_principal, target_path) return posix_path - def render_uri_path_with_principal(self, path: str) -> PurePosixPath: + def render_uri_path_with_principal(self, path: str) -> Path: # Special case that shall be handled here, when path is '//' if path == "storage://": - return PosixPath(PLATFORM_DELIMITER) + return Path(PLATFORM_DELIMITER) # Normal processing flow path_url = urlparse(path, scheme="file") diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index 6c877e164..771ed4308 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -47,7 +47,7 @@ class TestRetriesExceeded(Exception): def run_async(coro): def wrapper(*args, **kwargs): - if sys.platform == 'win32': + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) return run(coro(*args, **kwargs)) From 6877b431c2657fc205c723add2c4fdf7ef96438e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 25 Feb 2019 21:46:04 +0200 Subject: [PATCH 36/49] Skip image ops on Windows --- python/tests/e2e/test_e2e_images.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/tests/e2e/test_e2e_images.py b/python/tests/e2e/test_e2e_images.py index a5e1fc2ea..35834da74 100644 --- a/python/tests/e2e/test_e2e_images.py +++ b/python/tests/e2e/test_e2e_images.py @@ -1,3 +1,4 @@ +import sys from pathlib import Path from uuid import uuid4 as uuid @@ -44,6 +45,9 @@ async def image(loop, docker, tag): @pytest.mark.e2e +@pytest.mark.skipif( + sys.platform == "win32", reason="Image operations are not supported on Windows yet" +) def test_images_complete_lifecycle(helper, run_cli, image, tag, loop, docker): # Let`s push image captured = run_cli(["image", "push", image]) From 258af1a2137ec50667be755bb776cb44b676140c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 26 Feb 2019 17:17:25 +0200 Subject: [PATCH 37/49] Work on --- python/tests/e2e/conftest.py | 12 +++++++++++- python/tests/e2e/test_e2e.py | 2 -- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index 771ed4308..84298aeab 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -45,11 +45,21 @@ class TestRetriesExceeded(Exception): SysCap = namedtuple("SysCap", "out err") +async def _run_async(coro, *args, **kwargs): + try: + return await coro(*args, **kwargs) + finally: + if sys.platform == 'win32': + await asyncio.sleep(0.2) + else: + await asyncio.sleep(0.05) + + def run_async(coro): def wrapper(*args, **kwargs): if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) - return run(coro(*args, **kwargs)) + return run(_run_async(coro, *args, **kwargs)) return wrapper diff --git a/python/tests/e2e/test_e2e.py b/python/tests/e2e/test_e2e.py index acd2eeb43..631193699 100644 --- a/python/tests/e2e/test_e2e.py +++ b/python/tests/e2e/test_e2e.py @@ -43,8 +43,6 @@ def test_empty_directory_ls_output(run_cli, helper): # Ensure output of ls - empty directory shall print nothing. captured = run_cli(["storage", "ls", helper.tmpstorage]) assert not captured.out - # FIXME: stderr has "Using path ..." line - assert len(captured.err.splitlines()) == 1 and captured.err.startswith("Using path") @pytest.mark.e2e From 587cc5ea41346d959177bce9073d462f251c4c75 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 26 Feb 2019 17:24:54 +0200 Subject: [PATCH 38/49] Reformat --- python/tests/e2e/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index 84298aeab..c0bcc988d 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -49,7 +49,7 @@ async def _run_async(coro, *args, **kwargs): try: return await coro(*args, **kwargs) finally: - if sys.platform == 'win32': + if sys.platform == "win32": await asyncio.sleep(0.2) else: await asyncio.sleep(0.05) From b128ae20eb208e03c3b4df61804d9a53966d8aa8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 27 Feb 2019 12:48:36 +0200 Subject: [PATCH 39/49] Disable another non-windows test --- python/tests/e2e/test_e2e_images.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/tests/e2e/test_e2e_images.py b/python/tests/e2e/test_e2e_images.py index 7f90b7cfd..b169c9d2e 100644 --- a/python/tests/e2e/test_e2e_images.py +++ b/python/tests/e2e/test_e2e_images.py @@ -123,6 +123,9 @@ def check_job_output(): @pytest.mark.e2e +@pytest.mark.skipif( + sys.platform == "win32", reason="Image operations are not supported on Windows yet" +) def test_images_push_with_specified_name(helper, run_cli, image, tag, loop, docker): # Let`s push image image_no_tag = image.replace(f":{tag}", "") From 46e4735a474b8d5b6870ca4899c9e6f1f1a3ec59 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 27 Feb 2019 16:20:27 +0200 Subject: [PATCH 40/49] Fix a error --- python/tests/e2e/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index 91451b089..fbf0a3ed4 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -349,6 +349,7 @@ def _temp_config(): dir=self._tmp, prefix="run_cli-", suffix=".nmrc", delete=False ) config_path = Path(config_file.name) + del config_file # close file, it prevents unlink() error on Windows rc.save(config_path, self._config) return config_path From 9900ffcd321385409d9108318162d48e8f8fc100 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 27 Feb 2019 18:49:37 +0200 Subject: [PATCH 41/49] Make everything except recursive copy test working --- python/Makefile | 2 +- python/tests/e2e/conftest.py | 9 +++++---- python/tests/e2e/test_e2e.py | 3 +++ python/tests/e2e/test_e2e_jobs.py | 5 +++++ python/tests/e2e/test_e2e_network.py | 3 +++ 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/python/Makefile b/python/Makefile index 2497d9e5c..c8fbfe6c7 100644 --- a/python/Makefile +++ b/python/Makefile @@ -93,7 +93,7 @@ _e2e: .PHONY: _e2e_win _e2e_win: pytest \ - -m "e2e" \ + -m "e2e and not no_win32" \ --cov=neuromation \ --cov-report term-missing:skip-covered \ --cov-report xml:coverage.xml \ diff --git a/python/tests/e2e/conftest.py b/python/tests/e2e/conftest.py index fbf0a3ed4..73d61e459 100644 --- a/python/tests/e2e/conftest.py +++ b/python/tests/e2e/conftest.py @@ -345,11 +345,12 @@ async def _check_job_output(): def run_cli(self, arguments: List[str], storage_retry: bool = True) -> SysCap: def _temp_config(): - config_file = tempfile.NamedTemporaryFile( + with tempfile.NamedTemporaryFile( dir=self._tmp, prefix="run_cli-", suffix=".nmrc", delete=False - ) - config_path = Path(config_file.name) - del config_file # close file, it prevents unlink() error on Windows + ) as config_file: + # close tmp file on exit from context manager, + # it prevents unlink() error on Windows + config_path = Path(config_file.name) rc.save(config_path, self._config) return config_path diff --git a/python/tests/e2e/test_e2e.py b/python/tests/e2e/test_e2e.py index 99740acbf..40dd769fd 100644 --- a/python/tests/e2e/test_e2e.py +++ b/python/tests/e2e/test_e2e.py @@ -46,6 +46,7 @@ def test_empty_directory_ls_output(helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_e2e_job_top(helper): def split_non_empty_parts(line, separator=None): return [part.strip() for part in line.split(separator) if part.strip()] @@ -99,6 +100,7 @@ def split_non_empty_parts(line, separator=None): @pytest.mark.e2e +@pytest.mark.no_win32 @pytest.mark.parametrize( "switch,expected", [["--extshm", True], ["--no-extshm", False], [None, True]], # default is enabled @@ -160,6 +162,7 @@ def test_e2e_storage(data, tmp_path, helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_job_storage_interaction(helper, data, tmp_path): srcfile, checksum = data # Create directory for the test diff --git a/python/tests/e2e/test_e2e_jobs.py b/python/tests/e2e/test_e2e_jobs.py index 956771928..8a8379b43 100644 --- a/python/tests/e2e/test_e2e_jobs.py +++ b/python/tests/e2e/test_e2e_jobs.py @@ -166,6 +166,7 @@ def test_job_description(helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_unschedulable_job_lifecycle(helper): # Remember original running jobs captured = helper.run_cli( @@ -231,6 +232,7 @@ def test_unschedulable_job_lifecycle(helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_two_jobs_at_once(helper): # Remember original running jobs captured = helper.run_cli( @@ -396,6 +398,7 @@ async def get_(url): @pytest.mark.e2e +@pytest.mark.no_win32 def test_model_without_command(helper): loop_sleep = 1 service_wait_time = 60 @@ -451,6 +454,7 @@ async def get_(url): @pytest.mark.e2e +@pytest.mark.no_win32 def test_e2e_no_env(helper): bash_script = 'echo "begin"$VAR"end" | grep beginend' command = f"bash -c '{bash_script}'" @@ -815,6 +819,7 @@ def test_e2e_ssh_exec_dead_job(helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_e2e_job_list_filtered_by_status(helper): N_JOBS = 5 diff --git a/python/tests/e2e/test_e2e_network.py b/python/tests/e2e/test_e2e_network.py index 44a013ff3..519ccee16 100644 --- a/python/tests/e2e/test_e2e_network.py +++ b/python/tests/e2e/test_e2e_network.py @@ -42,6 +42,7 @@ def go(http_port: bool): @pytest.mark.e2e +@pytest.mark.no_win32 def test_connectivity_job_with_http_port(secret_job, check_http_get, helper): http_job = secret_job(True) @@ -76,6 +77,7 @@ def test_connectivity_job_with_http_port(secret_job, check_http_get, helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_connectivity_job_without_http_port(secret_job, check_http_get, helper): # run http job for getting url http_job = secret_job(True) @@ -114,6 +116,7 @@ def test_connectivity_job_without_http_port(secret_job, check_http_get, helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_check_isolation(secret_job, helper_alt): http_job = secret_job(True) From f66ccff24708f7f24cbc401eedfbec1217d51f33 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 10:21:31 +0200 Subject: [PATCH 42/49] Fix path mornalization for neuro cp --- python/neuromation/cli/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/neuromation/cli/storage.py b/python/neuromation/cli/storage.py index 460e0f8d4..ddacf4182 100644 --- a/python/neuromation/cli/storage.py +++ b/python/neuromation/cli/storage.py @@ -133,9 +133,9 @@ async def cp( progress_obj = ProgressBase.create_progress(progress) # len(uri.scheme) == 1 is a workaround for Windows path like C:/path/to.txt if not src.scheme or len(src.scheme) == 1: - src = URL(f"file:{src.path}") + src = URL(f"file:{source}") if not dst.scheme or len(dst.scheme) == 1: - dst = URL(f"file:{dst.path}") + dst = URL(f"file:{destination}") async with cfg.make_client(timeout=timeout) as client: if src.scheme == "file" and dst.scheme == "storage": src = normalize_local_path_uri(src) From 13759e54b9cb7d83fc922aab649b7cef99803d1a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 12:32:26 +0200 Subject: [PATCH 43/49] Fix test --- python/tests/e2e/test_e2e_copy_recursive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/tests/e2e/test_e2e_copy_recursive.py b/python/tests/e2e/test_e2e_copy_recursive.py index 9533626cf..df69d3d3a 100644 --- a/python/tests/e2e/test_e2e_copy_recursive.py +++ b/python/tests/e2e/test_e2e_copy_recursive.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from tests.e2e.utils import FILE_SIZE_B @@ -6,7 +8,7 @@ @pytest.mark.e2e def test_e2e_copy_recursive_to_platform(helper, nested_data, tmp_path): srcfile, checksum, dir_path = nested_data - target_file_name = srcfile.split("/")[-1] + target_file_name = Path(srcfile).name # Upload local file captured = helper.run_cli(["storage", "cp", "-r", dir_path, helper.tmpstorage]) From 19fdcc85029b2c2628571e2697e057ecf930d8d9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 13:08:35 +0200 Subject: [PATCH 44/49] Install codecov plugin from PyPI --- python/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/Makefile b/python/Makefile index c8fbfe6c7..a8f5974af 100644 --- a/python/Makefile +++ b/python/Makefile @@ -165,7 +165,8 @@ publish: dist publish-lint .PHONY: coverage coverage: - $(SHELL) <(curl -s https://codecov.io/bash) -X coveragepy + pip install codecov + codecov .PHONY: format format: From a890c12d83d56b9ab509aa16e0cace949b687b4d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 13:17:33 +0200 Subject: [PATCH 45/49] Add codecov params --- python/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Makefile b/python/Makefile index a8f5974af..de48283e5 100644 --- a/python/Makefile +++ b/python/Makefile @@ -166,7 +166,7 @@ publish: dist publish-lint .PHONY: coverage coverage: pip install codecov - codecov + codecov -f coverage.xml -X gcov .PHONY: format format: From 7966cc2e8451a4b3c2d6f2eab3b0e177c4198f00 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 13:18:22 +0200 Subject: [PATCH 46/49] Combine coverage reports --- python/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/python/Makefile b/python/Makefile index de48283e5..0a6e5b093 100644 --- a/python/Makefile +++ b/python/Makefile @@ -166,6 +166,7 @@ publish: dist publish-lint .PHONY: coverage coverage: pip install codecov + coverage combine codecov -f coverage.xml -X gcov .PHONY: format From de0a7ef59abe49cc6246f60c011e05d3186dcfc4 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 13:35:24 +0200 Subject: [PATCH 47/49] Fix settings --- python/Makefile | 1 - python/pytest.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/Makefile b/python/Makefile index 0a6e5b093..de48283e5 100644 --- a/python/Makefile +++ b/python/Makefile @@ -166,7 +166,6 @@ publish: dist publish-lint .PHONY: coverage coverage: pip install codecov - coverage combine codecov -f coverage.xml -X gcov .PHONY: format diff --git a/python/pytest.ini b/python/pytest.ini index 0ac4a4b71..b560fa7a6 100644 --- a/python/pytest.ini +++ b/python/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts= --cov-branch +addopts= --cov-branch --cov-report xml log_cli=false log_level=INFO filterwarnings=error From f1b621ef99a98aa0a95e63cc6d8e37fee6544622 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 14:07:12 +0200 Subject: [PATCH 48/49] Skip more tests on windows --- python/tests/e2e/test_e2e_jobs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/tests/e2e/test_e2e_jobs.py b/python/tests/e2e/test_e2e_jobs.py index 8a8379b43..48e4313d7 100644 --- a/python/tests/e2e/test_e2e_jobs.py +++ b/python/tests/e2e/test_e2e_jobs.py @@ -550,6 +550,7 @@ def test_e2e_env_from_local(helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_e2e_multiple_env(helper): bash_script = 'echo begin"$VAR""$VAR2"end | grep beginVALVAL2end' command = f"bash -c '{bash_script}'" @@ -623,6 +624,7 @@ def test_e2e_multiple_env_from_file(helper, tmp_path): @pytest.mark.e2e +@pytest.mark.no_win32 def test_e2e_ssh_exec_true(helper): command = 'bash -c "sleep 15m; false"' captured = helper.run_cli( @@ -676,6 +678,7 @@ def test_e2e_ssh_exec_false(helper): @pytest.mark.e2e +@pytest.mark.no_win32 def test_e2e_ssh_exec_no_cmd(helper): command = 'bash -c "sleep 15m; false"' captured = helper.run_cli( From 7b40002671f3d3635439dc1faeaf9b71c4a962ce Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 28 Feb 2019 14:30:09 +0200 Subject: [PATCH 49/49] Touch --- python/tests/e2e/test_e2e_copy_recursive.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/tests/e2e/test_e2e_copy_recursive.py b/python/tests/e2e/test_e2e_copy_recursive.py index df69d3d3a..98643e6b2 100644 --- a/python/tests/e2e/test_e2e_copy_recursive.py +++ b/python/tests/e2e/test_e2e_copy_recursive.py @@ -21,7 +21,6 @@ def test_e2e_copy_recursive_to_platform(helper, nested_data, tmp_path): ) # Download into local directory and confirm checksum - targetdir = tmp_path / "bar" targetdir.mkdir() helper.run_cli(["storage", "cp", "-r", f"{helper.tmpstorage}", str(targetdir)])