Implement FileStatus.uri property

CHANGELOG.D/1648.feature

@@ -0,0 +1 @@
+Implement `FileStatus.uri` property

neuromation/api/storage.py

@@ -73,6 +73,7 @@ class FileStatus:
     type: FileStatusType
     modification_time: int
     permission: Action
+    uri: URL
 
     def is_file(self) -> bool:
         return self.type == FileStatusType.FILE
@@ -93,10 +94,16 @@ def __init__(self, core: _Core, config: Config) -> None:
         self._min_time_diff = 0.0
         self._max_time_diff = 0.0
 
-    def _uri_to_path(self, uri: URL) -> str:
-        uri = normalize_storage_path_uri(
+    def _normalize_uri(self, uri: URL) -> URL:
+        return normalize_storage_path_uri(
             uri, self._config.username, self._config.cluster_name
         )
+
+    def _uri_to_path(self, uri: URL, *, normalized: bool = False) -> str:
+        if not normalized:
+            uri = normalize_storage_path_uri(
+                uri, self._config.username, self._config.cluster_name
+            )
         if not uri.host:
             return ""
         if uri.host != self._config.cluster_name:
@@ -133,22 +140,25 @@ def _is_remote_modified(self, local: os.stat_result, remote: FileStatus) -> bool
         )
 
     async def ls(self, uri: URL) -> AsyncIterator[FileStatus]:
-        url = self._config.storage_url / self._uri_to_path(uri)
+        uri = self._normalize_uri(uri)
+        url = self._config.storage_url / self._uri_to_path(uri, normalized=True)
         url = url.with_query(op="LISTSTATUS")
         headers = {"Accept": "application/x-ndjson"}
 
         request_time = time.time()
         auth = await self._config._api_auth()
+        # NB: the storage server returns file names in FileStatus for LISTSTATUS
+        # but full path for GETFILESTATUS
         async with self._core.request("GET", url, headers=headers, auth=auth) as resp:
             self._set_time_diff(request_time, resp)
             if resp.headers.get("Content-Type", "").startswith("application/x-ndjson"):
                 async for line in resp.content:
                     status = json.loads(line)["FileStatus"]
-                    yield _file_status_from_api(status)
+                    yield _file_status_from_api(uri, status)
             else:
                 res = await resp.json()
                 for status in res["FileStatuses"]["FileStatus"]:
-                    yield _file_status_from_api(status)
+                    yield _file_status_from_api(uri, status)
 
     async def glob(self, uri: URL, *, dironly: bool = False) -> AsyncIterator[URL]:
         if not _has_magic(uri.path):
@@ -254,15 +264,18 @@ async def create(self, uri: URL, data: Union[bytes, AsyncIterator[bytes]]) -> No
             resp  # resp.status == 201
 
     async def stat(self, uri: URL) -> FileStatus:
-        url = self._config.storage_url / self._uri_to_path(uri)
+        uri = self._normalize_uri(uri)
+        url = self._config.storage_url / self._uri_to_path(uri, normalized=True)
         url = url.with_query(op="GETFILESTATUS")
         auth = await self._config._api_auth()
 
         request_time = time.time()
+        # NB: the storage server returns file names in FileStatus for LISTSTATUS
+        # but full path for GETFILESTATUS
         async with self._core.request("GET", url, auth=auth) as resp:
             self._set_time_diff(request_time, resp)
             res = await resp.json()
-            return _file_status_from_api(res["FileStatus"])
+            return _file_status_from_api(None, res["FileStatus"])
 
     async def open(self, uri: URL) -> AsyncIterator[bytes]:
         url = self._config.storage_url / self._uri_to_path(uri)
@@ -693,13 +706,20 @@ def _isrecursive(pattern: str) -> bool:
     return pattern == "**"
 
 
-def _file_status_from_api(values: Dict[str, Any]) -> FileStatus:
+def _file_status_from_api(
+    base_uri: Optional[URL], values: Dict[str, Any]
+) -> FileStatus:
+    if base_uri is None:
+        uri = URL("storage://" + values["path"].lstrip("/"))
+    else:
+        uri = base_uri / values["path"]
     return FileStatus(
         path=values["path"],
         type=FileStatusType(values["type"]),
         size=int(values["length"]),
         modification_time=int(values["modificationTime"]),
         permission=Action(values["permission"]),
+        uri=uri,
     )
 
 

tests/api/test_storage.py

@@ -166,13 +166,15 @@ async def handler(request: web.Request) -> web.Response:
             type=FileStatusType.FILE,
             modification_time=0,
             permission=Action.READ,
+            uri=URL("storage://default/user/folder/foo"),
         ),
         FileStatus(
             path="bar",
             size=4 * 1024,
             type=FileStatusType.DIRECTORY,
             modification_time=0,
             permission=Action.READ,
+            uri=URL("storage://default/user/folder/bar"),
         ),
     ]
 
@@ -231,13 +233,15 @@ async def handler(request: web.Request) -> web.StreamResponse:
             type=FileStatusType.FILE,
             modification_time=0,
             permission=Action.READ,
+            uri=URL("storage://default/user/folder/foo"),
         ),
         FileStatus(
             path="bar",
             size=4 * 1024,
             type=FileStatusType.DIRECTORY,
             modification_time=0,
             permission=Action.READ,
+            uri=URL("storage://default/user/folder/bar"),
         ),
     ]
 
@@ -666,6 +670,42 @@ async def handler(request: web.Request) -> web.Response:
             size=1234,
             modification_time=3456,
             permission=Action.READ,
+            uri=URL("storage://default/user/folder"),
+        )
+
+
+async def test_storage_stats_user_home(
+    aiohttp_server: _TestServerFactory, make_client: _MakeClient
+) -> None:
+    async def handler(request: web.Request) -> web.Response:
+        assert request.path == "/storage/user"
+        assert request.query == {"op": "GETFILESTATUS"}
+        return web.json_response(
+            {
+                "FileStatus": {
+                    "path": "/default/user",
+                    "type": "DIRECTORY",
+                    "length": 1234,
+                    "modificationTime": 3456,
+                    "permission": "read",
+                }
+            }
+        )
+
+    app = web.Application()
+    app.router.add_get("/storage/user", handler)
+
+    srv = await aiohttp_server(app)
+
+    async with make_client(srv.make_url("/")) as client:
+        stats = await client.storage.stat(URL("storage:"))
+        assert stats == FileStatus(
+            path="/default/user",
+            type=FileStatusType.DIRECTORY,
+            size=1234,
+            modification_time=3456,
+            permission=Action.READ,
+            uri=URL("storage://default/user"),
         )
 
 
@@ -684,7 +724,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
             return web.json_response(
                 {
                     "FileStatus": {
-                        "path": "/user/file",
+                        "path": "file",
                         "type": "FILE",
                         "length": 5,
                         "modificationTime": 3456,

tests/api/test_storage_filestatus.py

@@ -1,16 +1,19 @@
+from yarl import URL
+
 from neuromation.api import FileStatusType
 from neuromation.api.storage import _file_status_from_api
 
 
 def test_from_api() -> None:
     stat = _file_status_from_api(
+        URL("storage://default/user/foo"),
         {
             "path": "name",
             "type": "FILE",
             "length": 1234,
             "modificationTime": 3456,
             "permission": "read",
-        }
+        },
     )
     assert stat.path == "name"
     assert stat.type == FileStatusType.FILE
@@ -21,13 +24,14 @@ def test_from_api() -> None:
 
 def test_file() -> None:
     stat = _file_status_from_api(
+        URL("storage://default/user/foo"),
         {
             "path": "name",
             "type": "FILE",
             "length": 1234,
             "modificationTime": 3456,
             "permission": "read",
-        }
+        },
     )
     assert stat.type == FileStatusType.FILE
     assert stat.is_file()
@@ -36,13 +40,14 @@ def test_file() -> None:
 
 def test_is_dir() -> None:
     stat = _file_status_from_api(
+        URL("storage://default/user/foo"),
         {
             "path": "name",
             "type": "DIRECTORY",
             "length": 1234,
             "modificationTime": 3456,
             "permission": "read",
-        }
+        },
     )
     assert stat.type == FileStatusType.DIRECTORY
     assert not stat.is_file()
@@ -51,12 +56,27 @@ def test_is_dir() -> None:
 
 def test_name() -> None:
     stat = _file_status_from_api(
+        URL("storage://default/user/foo"),
         {
             "path": "name",
             "type": "FILE",
             "length": 1234,
             "modificationTime": 3456,
             "permission": "read",
-        }
+        },
     )
     assert stat.name == "name"
+
+
+def test_uri() -> None:
+    stat = _file_status_from_api(
+        URL("storage://default/user/foo"),
+        {
+            "path": "name",
+            "type": "FILE",
+            "length": 1234,
+            "modificationTime": 3456,
+            "permission": "read",
+        },
+    )
+    assert stat.uri == URL("storage://default/user/foo/name")