Add support of project in disks (#2960)

* add support for project names in disks

* add custom project create disk test

* changelog

* revert usage of resolve_disk

* assume there is no owner in URIs

CHANGELOG.D/2960.feature

@@ -0,0 +1 @@
+Support projects in disk CLI commands and SDK methods.

CLI.md

@@ -2103,6 +2103,7 @@ Name | Description|
 |_--cluster CLUSTER_|Perform in a specified cluster \(the current cluster by default).|
 |_--name NAME_|Optional disk name|
 |_--org ORG_|Perform in a specified org \(the current org by default).|
+|_--project PROJECT_|Create disk in a specified project \(the current project by default).|
 |_\--timeout-unused TIMEDELTA_|Optional disk lifetime limit after last usage in the format '1d2h3m4s' \(some parts may be missing). Set '0' to disable. Default value '1d' can be changed in the user config.|
 
 
@@ -2147,6 +2148,7 @@ Name | Description|
 |_--cluster CLUSTER_|Look on a specified cluster \(the current cluster by default).|
 |_\--full-uri_|Output full disk URI.|
 |_\--long-format_|Output all info about disk.|
+|_--project PROJECT_|Look on a specified project \(all projects in current cluster by default).|
 
 
 

neuro-cli/docs/disk.md

@@ -64,6 +64,7 @@ $ neuro disk create 500M
 | _--cluster CLUSTER_ | Perform in a specified cluster \(the current cluster by default\). |
 | _--name NAME_ | Optional disk name |
 | _--org ORG_ | Perform in a specified org \(the current org by default\). |
+| _--project PROJECT_ | Create disk in a specified project \(the current project by default\). |
 | _--timeout-unused TIMEDELTA_ | Optional disk lifetime limit after last usage in the format '1d2h3m4s' \(some parts may be missing\). Set '0' to disable. Default value '1d' can be changed in the user config. |
 
 
@@ -112,6 +113,7 @@ List disks.
 | _--cluster CLUSTER_ | Look on a specified cluster \(the current cluster by default\). |
 | _--full-uri_ | Output full disk URI. |
 | _--long-format_ | Output all info about disk. |
+| _--project PROJECT_ | Look on a specified project \(all projects in current cluster by default\). |
 
 
 

neuro-cli/src/neuro_cli/blob_storage.py

@@ -96,7 +96,7 @@ async def lsbucket(
             uri_fmtr: URIFormatter = str
         else:
             uri_fmtr = uri_formatter(
-                username=root.client.username,
+                project_name=root.client.config.project_name_or_raise,
                 cluster_name=cluster or root.client.cluster_name,
                 org_name=root.client.config.org_name,
             )
@@ -365,7 +365,7 @@ async def statbucket(
         uri_fmtr: URIFormatter = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=cluster or root.client.cluster_name,
             org_name=root.client.config.org_name,
         )
@@ -550,7 +550,7 @@ async def ls(
         uri_fmtr: URIFormatter = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=root.client.cluster_name,
             org_name=root.client.config.org_name,
         )
@@ -606,7 +606,7 @@ async def glob(root: Root, full_uri: bool, patterns: Sequence[URL]) -> None:
         uri_fmtr: URIFormatter = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=root.client.cluster_name,
             org_name=root.client.config.org_name,
         )

neuro-cli/src/neuro_cli/disks.py

@@ -1,11 +1,14 @@
 from datetime import timedelta
-from typing import Optional, Sequence
+from typing import Optional, Sequence, Union
+
+from yarl import URL
 
 from neuro_cli.click_types import (
     CLUSTER,
     DISK,
     DISK_NAME,
     ORG,
+    PROJECT,
     PlatformURIType,
     UnionType,
 )
@@ -39,10 +42,19 @@ def disk() -> None:
     type=CLUSTER,
     help="Look on a specified cluster (the current cluster by default).",
 )
+@option(
+    "--project",
+    type=PROJECT,
+    help="Look on a specified project (all projects in current cluster by default).",
+)
 @option("--full-uri", is_flag=True, help="Output full disk URI.")
 @option("--long-format", is_flag=True, help="Output all info about disk.")
 async def ls(
-    root: Root, full_uri: bool, long_format: bool, cluster: Optional[str]
+    root: Root,
+    full_uri: bool,
+    long_format: bool,
+    cluster: Optional[str],
+    project: Optional[str],
 ) -> None:
     """
     List disks.
@@ -54,7 +66,7 @@ async def ls(
             uri_fmtr: URIFormatter = str
         else:
             uri_fmtr = uri_formatter(
-                username=root.client.username,
+                project_name=root.client.config.project_name_or_raise,
                 cluster_name=cluster or root.client.cluster_name,
                 org_name=root.client.config.org_name,
             )
@@ -66,7 +78,9 @@ async def ls(
 
     disks = []
     with root.status("Fetching disks") as status:
-        async with root.client.disks.list(cluster_name=cluster) as it:
+        async with root.client.disks.list(
+            cluster_name=cluster, project_name=project
+        ) as it:
             async for disk in it:
                 disks.append(disk)
                 status.update(f"Fetching disks ({len(disks)} loaded)")
@@ -105,13 +119,19 @@ async def ls(
     help="Optional disk name",
     default=None,
 )
+@option(
+    "--project",
+    type=PROJECT,
+    help="Create disk in a specified project (the current project by default).",
+)
 async def create(
     root: Root,
     storage: str,
     timeout_unused: Optional[str] = None,
     name: Optional[str] = None,
     cluster: Optional[str] = None,
     org: Optional[str] = None,
+    project: Optional[str] = None,
 ) -> None:
     """
     Create a disk
@@ -146,6 +166,7 @@ async def create(
         timeout_unused=disk_timeout_unused,
         name=name,
         cluster_name=cluster,
+        project_name=project,
         org_name=org_name,
     )
     disk_fmtr = DiskFormatter(
@@ -165,17 +186,20 @@ async def create(
     "disk", type=UnionType("disk", PlatformURIType(allowed_schemes=("disk",)), DISK)
 )
 @option("--full-uri", is_flag=True, help="Output full disk URI.")
-async def get(root: Root, cluster: Optional[str], disk: str, full_uri: bool) -> None:
+async def get(
+    root: Root, cluster: Optional[str], disk: Union[str, URL], full_uri: bool
+) -> None:
     """
     Get disk DISK_ID.
     """
     disk_id = await resolve_disk(disk, client=root.client, cluster_name=cluster)
     disk_obj = await root.client.disks.get(disk_id, cluster_name=cluster)
+
     if full_uri:
         uri_fmtr: URIFormatter = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=cluster or root.client.cluster_name,
             org_name=root.client.config.org_name,
         )

neuro-cli/src/neuro_cli/formatters/disks.py

@@ -103,6 +103,8 @@ def __call__(self, disk: Disk) -> RenderableType:
         if disk.name:
             table.add_row("Name", disk.name)
         table.add_row("Org name", disk.org_name or ORG.NO_ORG_STR)
+        table.add_row("Project name", disk.project_name)
+        table.add_row("Owner", disk.owner)
         table.add_row("Status", disk.status.value)
         table.add_row("Created at", self._datetime_formatter(disk.created_at))
         table.add_row("Last used", self._datetime_formatter(disk.last_usage))

neuro-cli/src/neuro_cli/formatters/utils.py

@@ -12,22 +12,22 @@
 
 
 def uri_formatter(
-    username: str, cluster_name: str, org_name: Optional[str]
+    project_name: str, cluster_name: str, org_name: Optional[str]
 ) -> URIFormatter:
     def formatter(uri: URL) -> str:
         if uri.scheme in SCHEMES:
             if uri.host == cluster_name:
                 assert uri.path[0] == "/"
                 path = uri.path.lstrip("/")
-                owner_or_org, _, rest = path.partition("/")
+                project_or_org, _, rest = path.partition("/")
                 if org_name:
-                    if owner_or_org != org_name:
+                    if project_or_org != org_name:
                         return str(uri)
                     path = rest
-                    owner, _, rest = path.partition("/")
+                    project, _, rest = path.partition("/")
                 else:
-                    owner = owner_or_org
-                if owner == username:
+                    project = project_or_org
+                if project == project_name:
                     path = rest.lstrip("/")
                 else:
                     path = "/" + path

neuro-cli/src/neuro_cli/image.py

@@ -154,7 +154,7 @@ async def ls(
         image_fmtr = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=root.client.cluster_name,
             org_name=root.client.config.org_name,
         )

neuro-cli/src/neuro_cli/job.py

@@ -411,7 +411,7 @@ async def ls(
         uri_fmtr = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=root.client.cluster_name,
             org_name=root.client.config.org_name,
         )
@@ -483,7 +483,7 @@ async def status(root: Root, job: str, full_uri: bool) -> None:
         uri_fmtr = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=root.client.cluster_name,
             org_name=root.client.config.org_name,
         )
@@ -745,7 +745,7 @@ async def renderer() -> None:
         uri_fmtr = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=root.client.cluster_name,
             org_name=root.client.config.org_name,
         )

neuro-cli/src/neuro_cli/secrets.py

@@ -38,7 +38,7 @@ async def ls(root: Root, full_uri: bool, cluster: Optional[str]) -> None:
             uri_fmtr: URIFormatter = str
         else:
             uri_fmtr = uri_formatter(
-                username=root.client.username,
+                project_name=root.client.config.project_name_or_raise,
                 cluster_name=cluster or root.client.cluster_name,
                 org_name=root.client.config.org_name,
             )

neuro-cli/src/neuro_cli/share.py

@@ -138,7 +138,7 @@ async def ls(
         uri_fmtr = str
     else:
         uri_fmtr = uri_formatter(
-            username=root.client.username,
+            project_name=root.client.config.project_name_or_raise,
             cluster_name=root.client.cluster_name,
             org_name=root.client.config.org_name,
         )

neuro-cli/src/neuro_cli/utils.py

@@ -521,7 +521,7 @@ async def resolve_disk(
                 return disk.id
             except ResourceNotFound:
                 pass
-        raise ValueError(f"Failed to resolve job {id_or_name_or_uri}")
+        raise ValueError(f"Failed to resolve disk {id_or_name_or_uri}")
     else:
         disk = await client.disks.get(id_or_name, cluster_name)
     return disk.id

neuro-cli/tests/unit/formatters/ascii/TestJobOutputFormatter.test_job_with_disk_volumes_short[human]_0.ref

@@ -4,15 +4,15 @@ Job                     test-job
  Cluster                 default                                                     
  Description             test job description                                        
  Status                  failed (ErrorReason)                                        
- Image                   image:test-image:sometag                                    
+ Image                   image:/test-user/test-image:sometag                         
  Command                 test-command                                                
  Priority                Normal                                                      
  Price (credits / hour)  15.0000                                                     
  Current cost            150.0000                                                    
  Resources                Memory  16.8 MB                                            
                           CPU         0.1                                            
  TTY                     False                                                       
- Disk volumes             /mnt/disk1  disk:disk1                           READONLY  
+ Disk volumes             /mnt/disk1  disk:/test-user/disk1                READONLY  
                           /mnt/disk2  disk:/otheruser/disk2                          
                           /mnt/disk3  disk://othercluster/otheruser/disk3            
  Http URL                http://local.host.test/                                     

neuro-cli/tests/unit/formatters/ascii/TestJobOutputFormatter.test_job_with_disk_volumes_short[iso]_0.ref

@@ -4,15 +4,15 @@ Job                     test-job
  Cluster                 default                                                     
  Description             test job description                                        
  Status                  failed (ErrorReason)                                        
- Image                   image:test-image:sometag                                    
+ Image                   image:/test-user/test-image:sometag                         
  Command                 test-command                                                
  Priority                Normal                                                      
  Price (credits / hour)  15.0000                                                     
  Current cost            150.0000                                                    
  Resources                Memory  16.8 MB                                            
                           CPU         0.1                                            
  TTY                     False                                                       
- Disk volumes             /mnt/disk1  disk:disk1                           READONLY  
+ Disk volumes             /mnt/disk1  disk:/test-user/disk1                READONLY  
                           /mnt/disk2  disk:/otheruser/disk2                          
                           /mnt/disk3  disk://othercluster/otheruser/disk3            
  Http URL                http://local.host.test/                                     

neuro-cli/tests/unit/formatters/ascii/TestJobOutputFormatter.test_job_with_environment[human]_0.ref

@@ -1,24 +1,24 @@
-Job                     test-job                 
- Name                    test-job-name            
- Owner                   test-user                
- Cluster                 default                  
- Description             test job description     
- Status                  failed (ErrorReason)     
- Image                   image:test-image:sometag 
- Command                 test-command             
- Priority                Normal                   
- Price (credits / hour)  15.0000                  
- Current cost            150.0000                 
- Resources                Memory  16.8 MB         
-                          CPU         0.1         
- TTY                     False                    
- Http URL                http://local.host.test/  
- Http port               80                       
- Http authentication     True                     
- Environment              ENV_NAME_1  __value1__  
-                          ENV_NAME_2  **value2**  
- Created                 Sep 25 2018 at 12:28     
- Started                 Sep 25 2018 at 12:28     
- Finished                Sep 25 2018 at 12:28     
- Exit code               123                      
+Job                     test-job                            
+ Name                    test-job-name                       
+ Owner                   test-user                           
+ Cluster                 default                             
+ Description             test job description                
+ Status                  failed (ErrorReason)                
+ Image                   image:/test-user/test-image:sometag 
+ Command                 test-command                        
+ Priority                Normal                              
+ Price (credits / hour)  15.0000                             
+ Current cost            150.0000                            
+ Resources                Memory  16.8 MB                    
+                          CPU         0.1                    
+ TTY                     False                               
+ Http URL                http://local.host.test/             
+ Http port               80                                  
+ Http authentication     True                                
+ Environment              ENV_NAME_1  __value1__             
+                          ENV_NAME_2  **value2**             
+ Created                 Sep 25 2018 at 12:28                
+ Started                 Sep 25 2018 at 12:28                
+ Finished                Sep 25 2018 at 12:28                
+ Exit code               123                                 
  Description             ErrorDesc