Support copying to/from non-regular files (character devices, named p…

…ipes, etc).

CHANGELOG.D/813.feature

@@ -0,0 +1,4 @@
+`neuro storage cp` now supports copying to/from non-regular files like character devices and named pipes. In particular this allows to output the file to the stdout or get the input from the stdin::
+
+    neuro storage cp storage://~/file.txt /dev/stdout
+    neuro storage cp /dev/stdin storage://~/file.txt

neuromation/api/storage.py

@@ -2,6 +2,7 @@
 import enum
 import errno
 import logging
+import os
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, AsyncIterator, Dict, List, Optional
@@ -174,9 +175,9 @@ async def _iterate_file(
         self, src: Path, *, progress: Optional[AbstractProgress] = None
     ) -> AsyncIterator[bytes]:
         loop = asyncio.get_event_loop()
-        if progress is not None:
-            progress.start(str(src), src.stat().st_size)
         with src.open("rb") as stream:
+            if progress is not None:
+                progress.start(str(src), os.stat(stream.fileno()).st_size)
             chunk = await loop.run_in_executor(None, stream.read, 1024 * 1024)
             pos = len(chunk)
             while chunk:
@@ -198,8 +199,6 @@ async def upload_file(
             raise FileNotFoundError(f"'{path}' does not exist")
         if path.is_dir():
             raise IsADirectoryError(f"'{path}' is a directory, use recursive copy")
-        if not path.is_file():
-            raise OSError(f"'{path}' should be a regular file")
         if not dst.name:
             # file:src/file.txt -> storage:dst/ ==> storage:dst/file.txt
             dst = dst / src.name
@@ -262,8 +261,6 @@ async def download_file(
         if path.exists():
             if path.is_dir():
                 path = path / src.name
-            elif not path.is_file():
-                raise OSError(f"{path} should be a regular file")
         loop = asyncio.get_event_loop()
         with path.open("wb") as stream:
             stat = await self.stats(src)

tests/api/test_storage.py

@@ -1,4 +1,5 @@
 import errno
+import os
 from filecmp import dircmp
 from pathlib import Path
 from shutil import copytree
@@ -518,12 +519,24 @@ async def test_storage_upload_dir_doesnt_exist(make_client: _MakeClient) -> None
             )
 
 
-async def test_storage_upload_not_a_file(make_client: _MakeClient) -> None:
-    async with make_client("https://example.com") as client:
-        with pytest.raises(OSError):
-            await client.storage.upload_file(
-                URL("file:///dev/random"), URL("storage://host/path/to")
-            )
+async def test_storage_upload_not_a_file(
+    storage_server: Any, make_client: _MakeClient, storage_path: Path
+) -> None:
+    file_path = Path(os.devnull).absolute()
+    target_path = storage_path / "file.txt"
+    progress = mock.Mock()
+
+    async with make_client(storage_server.make_url("/")) as client:
+        await client.storage.upload_file(
+            URL(file_path.as_uri()), URL("storage:file.txt"), progress=progress
+        )
+
+    uploaded = target_path.read_bytes()
+    assert uploaded == b""
+
+    progress.start.assert_called_with(str(file_path), 0)
+    progress.progress.assert_not_called()
+    progress.complete.assert_called_with(str(file_path))
 
 
 async def test_storage_upload_regular_file_to_existing_file_target(
@@ -784,10 +797,9 @@ async def test_storage_download_regular_file_to_non_file(
     storage_file.write_bytes(src_file.read_bytes())
 
     async with make_client(storage_server.make_url("/")) as client:
-        with pytest.raises(OSError):
-            await client.storage.download_file(
-                URL("storage:file.txt"), URL("file:///dev/null")
-            )
+        await client.storage.download_file(
+            URL("storage:file.txt"), URL(Path(os.devnull).absolute().as_uri())
+        )
 
 
 async def test_storage_download_dir(