Support copying to/from non-regular files (character devices, named pipes, etc).

CHANGELOG.D/813.feature

@@ -0,0 +1 @@
+`neuro storage cp` now supports copying to/from non-regular files like character devices and named pipes. In particular this allows to output the file to the stdout or get the input from the stdin (`/dev/stdout` and `/dev/stdin` on Linux, `CON` on Windows).

neuromation/api/storage.py

@@ -2,6 +2,7 @@
 import enum
 import errno
 import logging
+import os
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, AsyncIterator, Dict, List, Optional
@@ -174,9 +175,9 @@ async def _iterate_file(
         self, src: Path, *, progress: Optional[AbstractProgress] = None
     ) -> AsyncIterator[bytes]:
         loop = asyncio.get_event_loop()
-        if progress is not None:
-            progress.start(str(src), src.stat().st_size)
         with src.open("rb") as stream:
+            if progress is not None:
+                progress.start(str(src), os.stat(stream.fileno()).st_size)
             chunk = await loop.run_in_executor(None, stream.read, 1024 * 1024)
             pos = len(chunk)
             while chunk:
@@ -194,14 +195,18 @@ async def upload_file(
         src = normalize_local_path_uri(src)
         dst = normalize_storage_path_uri(dst, self._config.auth_token.username)
         path = _extract_path(src)
-        if not path.exists():
-            raise FileNotFoundError(errno.ENOENT, "No such file", str(path))
-        if path.is_dir():
-            raise IsADirectoryError(
-                errno.EISDIR, "Is a directory, use recursive copy", str(path)
-            )
-        if not path.is_file():
-            raise OSError(errno.EINVAL, "Not a regular file", str(path))
+        try:
+            if not path.exists():
+                raise FileNotFoundError(errno.ENOENT, "No such file", str(path))
+            if path.is_dir():
+                raise IsADirectoryError(
+                    errno.EISDIR, "Is a directory, use recursive copy", str(path)
+                )
+        except OSError as e:
+            if getattr(e, "winerror", None) not in (1, 87):
+                raise
+            # Ignore stat errors for device files like NUL or CON on Windows.
+            # See https://bugs.python.org/issue37074
         if not dst.name:
             # file:src/file.txt -> storage:dst/ ==> storage:dst/file.txt
             dst = dst / src.name
@@ -265,11 +270,16 @@ async def download_file(
         src = normalize_storage_path_uri(src, self._config.auth_token.username)
         dst = normalize_local_path_uri(dst)
         path = _extract_path(dst)
-        if path.exists():
+        try:
             if path.is_dir():
                 path = path / src.name
-            elif not path.is_file():
-                raise OSError(errno.EINVAL, "Not a regular file", str(path))
+        except FileNotFoundError:
+            pass
+        except OSError as e:
+            if getattr(e, "winerror", None) not in (1, 87):
+                raise
+            # Ignore stat errors for device files like NUL or CON on Windows.
+            # See https://bugs.python.org/issue37074
         loop = asyncio.get_event_loop()
         with path.open("wb") as stream:
             stat = await self.stats(src)

tests/api/test_storage.py

@@ -1,4 +1,5 @@
 import errno
+import os
 from filecmp import dircmp
 from pathlib import Path
 from shutil import copytree
@@ -518,12 +519,24 @@ async def test_storage_upload_dir_doesnt_exist(make_client: _MakeClient) -> None
             )
 
 
-async def test_storage_upload_not_a_file(make_client: _MakeClient) -> None:
-    async with make_client("https://example.com") as client:
-        with pytest.raises(OSError):
-            await client.storage.upload_file(
-                URL("file:///dev/random"), URL("storage://host/path/to")
-            )
+async def test_storage_upload_not_a_file(
+    storage_server: Any, make_client: _MakeClient, storage_path: Path
+) -> None:
+    file_path = Path(os.devnull).absolute()
+    target_path = storage_path / "file.txt"
+    progress = mock.Mock()
+
+    async with make_client(storage_server.make_url("/")) as client:
+        await client.storage.upload_file(
+            URL(file_path.as_uri()), URL("storage:file.txt"), progress=progress
+        )
+
+    uploaded = target_path.read_bytes()
+    assert uploaded == b""
+
+    progress.start.assert_called_with(str(file_path), 0)
+    progress.progress.assert_not_called()
+    progress.complete.assert_called_with(str(file_path))
 
 
 async def test_storage_upload_regular_file_to_existing_file_target(
@@ -784,10 +797,9 @@ async def test_storage_download_regular_file_to_non_file(
     storage_file.write_bytes(src_file.read_bytes())
 
     async with make_client(storage_server.make_url("/")) as client:
-        with pytest.raises(OSError):
-            await client.storage.download_file(
-                URL("storage:file.txt"), URL("file:///dev/null")
-            )
+        await client.storage.download_file(
+            URL("storage:file.txt"), URL(Path(os.devnull).absolute().as_uri())
+        )
 
 
 async def test_storage_download_dir(