From 42984d92ed00f3e77170911d3932a8f6bc26dcb6 Mon Sep 17 00:00:00 2001 From: Ivan Zubenko Date: Fri, 5 May 2023 10:31:37 +0300 Subject: [PATCH] support projects in blob commands --- CHANGELOG.D/2954.feature | 1 + CLI.md | 18 +- neuro-cli/docs/blob.md | 18 +- neuro-cli/src/neuro_cli/blob_storage.py | 165 ++++++++++++++---- neuro-cli/src/neuro_cli/click_types.py | 45 +++-- neuro-cli/src/neuro_cli/formatters/buckets.py | 3 + neuro-cli/src/neuro_cli/utils.py | 6 +- neuro-cli/tests/e2e/conftest.py | 13 +- neuro-cli/tests/unit/conftest.py | 2 +- ...tter.test_long_formatter[formatter0]_0.ref | 2 +- ...tter.test_long_formatter[formatter0]_1.ref | 2 +- ...tter.test_long_formatter[formatter0]_2.ref | 2 +- ...tter.test_long_formatter[formatter0]_3.ref | 2 +- ...tter.test_long_formatter[formatter0]_4.ref | 2 +- ...tter.test_long_formatter[formatter0]_5.ref | 2 +- ...tter.test_long_formatter[formatter0]_6.ref | 2 +- ...tter.test_long_formatter[formatter0]_7.ref | 2 +- ...tter.test_long_formatter[formatter0]_8.ref | 2 +- ...tter.test_long_formatter[formatter1]_0.ref | 2 +- ...tter.test_long_formatter[formatter1]_1.ref | 2 +- ...tter.test_long_formatter[formatter1]_2.ref | 2 +- ...tter.test_long_formatter[formatter1]_3.ref | 2 +- ...tter.test_long_formatter[formatter1]_4.ref | 2 +- ...tter.test_long_formatter[formatter1]_5.ref | 2 +- ...tter.test_long_formatter[formatter1]_6.ref | 2 +- ...tter.test_long_formatter[formatter1]_7.ref | 2 +- ...tter.test_long_formatter[formatter1]_8.ref | 2 +- .../ascii/test_bucket_formatter_0.ref | 17 +- .../test_bucket_formatter_with_org_0.ref | 17 +- .../ascii/test_buckets_formatter_long_0.ref | 12 +- .../ascii/test_buckets_formatter_short_0.ref | 12 +- .../unit/formatters/test_blob_formatters.py | 4 +- .../formatters/test_bucket_credentials.py | 5 + .../tests/unit/formatters/test_buckets.py | 6 + neuro-cli/tests/unit/test_shell_completion.py | 80 +++++++++ neuro-sdk/src/neuro_sdk/_bucket_base.py | 3 +- neuro-sdk/src/neuro_sdk/_buckets.py | 108 +++++++++--- neuro-sdk/src/neuro_sdk/_parser.py | 2 +- neuro-sdk/tests/test_blob_storage.py | 1 + neuro-sdk/tests/test_buckets.py | 20 +++ 40 files changed, 457 insertions(+), 137 deletions(-) create mode 100644 CHANGELOG.D/2954.feature diff --git a/CHANGELOG.D/2954.feature b/CHANGELOG.D/2954.feature new file mode 100644 index 000000000..63ca47f0f --- /dev/null +++ b/CHANGELOG.D/2954.feature @@ -0,0 +1 @@ +Support projects in `neuro blob` cli commands and `neuro-sdk`. diff --git a/CLI.md b/CLI.md index 23687ee1a..f7b13db2a 100644 --- a/CLI.md +++ b/CLI.md @@ -1393,7 +1393,8 @@ Name | Description| |----|------------| |_--help_|Show this message and exit.| |_--cluster CLUSTER_|Look on a specified cluster \(the current cluster by default).| -|_--owner TEXT_|Owner of bucket to assume for named bucket \(the current user by default)| +|_--org ORG_|Look on a specified org \(the current org by default).| +|_--project PROJECT_|Look on a specified project \(the current project by default).| @@ -1444,6 +1445,7 @@ Name | Description| |_\--gcp-sa-credential GCP\_SA_CREDNETIAL_|GCP service account credential in form of base64 encoded json string that grants access to imported bucket. Required when PROVIDER is 'gcp'| |_--name NAME_|Optional bucket name| |_--org ORG_|Perform in a specified org \(the current org by default).| +|_--project PROJECT_|Perform in a specified project \(the current project by default).| |_--provider PROVIDER_|Bucket provider that hosts bucket \[required]| |_\--provider-bucket-name EXTERNAL_NAME_|Name of bucket \(or container in case of Azure) inside the provider \[required]| @@ -1491,6 +1493,8 @@ Name | Description| |_--cluster CLUSTER_|Look on a specified cluster \(the current cluster by default).| |_\--full-uri_|Output full bucket URI.| |_\--long-format_|Output all info about bucket.| +|_--org ORG_|Look on a specified org \(the current org by default).| +|_--project PROJECT_|Look on a specified project \(the current project by default).| @@ -1533,6 +1537,7 @@ Name | Description| |_--cluster CLUSTER_|Perform in a specified cluster \(the current cluster by default).| |_--name NAME_|Optional bucket name| |_--org ORG_|Perform in a specified org \(the current org by default).| +|_--project PROJECT_|Perform in a specified project \(the current project by default).| @@ -1554,6 +1559,8 @@ Name | Description| |_--help_|Show this message and exit.| |_--cluster CLUSTER_|Perform in a specified cluster \(the current cluster by default).| |_--name NAME_|Optional bucket credential name| +|_--org ORG_|Perform in a specified org \(the current org by default).| +|_--project PROJECT_|Perform in a specified project \(the current project by default).| |_\--read-only_|Make read-only credential| @@ -1598,7 +1605,8 @@ Name | Description| |_--help_|Show this message and exit.| |_--cluster CLUSTER_|Perform on a specified cluster \(the current cluster by default).| |_\-f, --force_|Force removal of all blobs inside bucket| -|_--owner TEXT_|Owner of bucket to assume for named bucket \(the current user by default)| +|_--org ORG_|Perform on a specified org \(the current org by default).| +|_--project PROJECT_|Perform on a specified project \(the current project by default).| @@ -1648,7 +1656,8 @@ Name | Description| |----|------------| |_--help_|Show this message and exit.| |_--cluster CLUSTER_|Perform on a specified cluster \(the current cluster by default).| -|_--owner TEXT_|Owner of bucket to assume for named bucket \(the current user by default)| +|_--org ORG_|Perform on a specified org \(the current org by default).| +|_--project PROJECT_|Perform on a specified project \(the current project by default).| @@ -1690,7 +1699,8 @@ Name | Description| |_--help_|Show this message and exit.| |_--cluster CLUSTER_|Look on a specified cluster \(the current cluster by default).| |_\--full-uri_|Output full bucket URI.| -|_--owner TEXT_|Owner of bucket to assume for named bucket \(the current user by default)| +|_--org ORG_|Look on a specified org \(the current org by default).| +|_--project PROJECT_|Look on a specified project \(the current project by default).| diff --git a/neuro-cli/docs/blob.md b/neuro-cli/docs/blob.md index ef3c7727c..2c7951bde 100644 --- a/neuro-cli/docs/blob.md +++ b/neuro-cli/docs/blob.md @@ -108,7 +108,8 @@ Get storage usage for `BUCKET`. | :--- | :--- | | _--help_ | Show this message and exit. | | _--cluster CLUSTER_ | Look on a specified cluster \(the current cluster by default\). | -| _--owner TEXT_ | Owner of bucket to assume for named bucket \(the current user by default\) | +| _--org ORG_ | Look on a specified org \(the current org by default\). | +| _--project PROJECT_ | Look on a specified project \(the current project by default\). | @@ -163,6 +164,7 @@ Import an existing bucket. | _--gcp-sa-credential GCP\_SA\_CREDNETIAL_ | GCP service account credential in form of base64 encoded json string that grants access to imported bucket. Required when PROVIDER is 'gcp' | | _--name NAME_ | Optional bucket name | | _--org ORG_ | Perform in a specified org \(the current org by default\). | +| _--project PROJECT_ | Perform in a specified project \(the current project by default\). | | _--provider PROVIDER_ | Bucket provider that hosts bucket _\[required\]_ | | _--provider-bucket-name EXTERNAL\_NAME_ | Name of bucket \(or container in case of Azure\) inside the provider _\[required\]_ | @@ -214,6 +216,8 @@ List buckets. | _--cluster CLUSTER_ | Look on a specified cluster \(the current cluster by default\). | | _--full-uri_ | Output full bucket URI. | | _--long-format_ | Output all info about bucket. | +| _--org ORG_ | Look on a specified org \(the current org by default\). | +| _--project PROJECT_ | Look on a specified project \(the current project by default\). | @@ -260,6 +264,7 @@ Create a new bucket. | _--cluster CLUSTER_ | Perform in a specified cluster \(the current cluster by default\). | | _--name NAME_ | Optional bucket name | | _--org ORG_ | Perform in a specified org \(the current org by default\). | +| _--project PROJECT_ | Perform in a specified project \(the current project by default\). | @@ -283,6 +288,8 @@ Create a new bucket credential. | _--help_ | Show this message and exit. | | _--cluster CLUSTER_ | Perform in a specified cluster \(the current cluster by default\). | | _--name NAME_ | Optional bucket credential name | +| _--org ORG_ | Perform in a specified org \(the current org by default\). | +| _--project PROJECT_ | Perform in a specified project \(the current project by default\). | | _--read-only_ | Make read-only credential | @@ -331,7 +338,8 @@ Remove bucket `BUCKET`. | _--help_ | Show this message and exit. | | _--cluster CLUSTER_ | Perform on a specified cluster \(the current cluster by default\). | | _-f, --force_ | Force removal of all blobs inside bucket | -| _--owner TEXT_ | Owner of bucket to assume for named bucket \(the current user by default\) | +| _--org ORG_ | Perform on a specified org \(the current org by default\). | +| _--project PROJECT_ | Perform on a specified project \(the current project by default\). | @@ -384,7 +392,8 @@ $ neuro blob set-bucket-publicity my-bucket private | :--- | :--- | | _--help_ | Show this message and exit. | | _--cluster CLUSTER_ | Perform on a specified cluster \(the current cluster by default\). | -| _--owner TEXT_ | Owner of bucket to assume for named bucket \(the current user by default\) | +| _--org ORG_ | Perform on a specified org \(the current org by default\). | +| _--project PROJECT_ | Perform on a specified project \(the current project by default\). | @@ -430,7 +439,8 @@ Get bucket `BUCKET`. | _--help_ | Show this message and exit. | | _--cluster CLUSTER_ | Look on a specified cluster \(the current cluster by default\). | | _--full-uri_ | Output full bucket URI. | -| _--owner TEXT_ | Owner of bucket to assume for named bucket \(the current user by default\) | +| _--org ORG_ | Look on a specified org \(the current org by default\). | +| _--project PROJECT_ | Look on a specified project \(the current project by default\). | diff --git a/neuro-cli/src/neuro_cli/blob_storage.py b/neuro-cli/src/neuro_cli/blob_storage.py index 30be85915..570d12306 100644 --- a/neuro-cli/src/neuro_cli/blob_storage.py +++ b/neuro-cli/src/neuro_cli/blob_storage.py @@ -22,6 +22,7 @@ BUCKET_NAME, CLUSTER, ORG, + PROJECT, PlatformURIType, ) from neuro_cli.formatters.bucket_credentials import ( @@ -81,10 +82,25 @@ def blob_storage() -> None: type=CLUSTER, help="Look on a specified cluster (the current cluster by default).", ) +@option( + "--org", + type=ORG, + help="Look on a specified org (the current org by default).", +) +@option( + "--project", + type=PROJECT, + help="Look on a specified project (the current project by default).", +) @option("--full-uri", is_flag=True, help="Output full bucket URI.") @option("--long-format", is_flag=True, help="Output all info about bucket.") async def lsbucket( - root: Root, full_uri: bool, long_format: bool, cluster: Optional[str] + root: Root, + full_uri: bool, + long_format: bool, + cluster: Optional[str], + org: Optional[str], + project: Optional[str], ) -> None: """ List buckets. @@ -106,9 +122,12 @@ async def lsbucket( long_format=long_format, ) + org_name = parse_org_name(org, root) buckets = [] with root.status("Fetching buckets") as status: - async with root.client.buckets.list(cluster_name=cluster) as it: + async with root.client.buckets.list( + cluster_name=cluster, org_name=org_name, project_name=project + ) as it: async for bucket in it: buckets.append(bucket) status.update(f"Fetching buckets ({len(buckets)} loaded)") @@ -128,6 +147,11 @@ async def lsbucket( type=ORG, help="Perform in a specified org (the current org by default).", ) +@option( + "--project", + type=PROJECT, + help="Perform in a specified project (the current project by default).", +) @option( "--name", type=BUCKET_NAME, @@ -140,6 +164,7 @@ async def mkbucket( name: Optional[str] = None, cluster: Optional[str] = None, org: Optional[str] = None, + project: Optional[str] = None, ) -> None: """ Create a new bucket. @@ -149,6 +174,7 @@ async def mkbucket( name=name, cluster_name=cluster, org_name=org_name, + project_name=project, ) bucket_fmtr = BucketFormatter( str, datetime_formatter=get_datetime_formatter(root.iso_datetime_format) @@ -168,6 +194,11 @@ async def mkbucket( type=ORG, help="Perform in a specified org (the current org by default).", ) +@option( + "--project", + type=PROJECT, + help="Perform in a specified project (the current project by default).", +) @option( "--name", type=BUCKET_NAME, @@ -270,6 +301,7 @@ async def importbucket( name: Optional[str] = None, cluster: Optional[str] = None, org: Optional[str] = None, + project: Optional[str] = None, ) -> None: """ Import an existing bucket. @@ -320,6 +352,7 @@ async def importbucket( name=name, cluster_name=cluster, org_name=org_name, + project_name=project, ) bucket_fmtr = BucketFormatter( str, datetime_formatter=get_datetime_formatter(root.iso_datetime_format) @@ -335,28 +368,41 @@ async def importbucket( help="Look on a specified cluster (the current cluster by default).", ) @option( - "--owner", - type=str, - help="Owner of bucket to assume for named bucket (the current user by default)", + "--org", + type=ORG, + help="Look on a specified org (the current org by default).", +) +@option( + "--project", + type=PROJECT, + help="Look on a specified project (the current project by default).", ) @argument("bucket", type=BUCKET) @option("--full-uri", is_flag=True, help="Output full bucket URI.") async def statbucket( root: Root, cluster: Optional[str], - owner: Optional[str], + org: Optional[str], + project: Optional[str], bucket: str, full_uri: bool, ) -> None: """ Get bucket BUCKET. """ + org_name = parse_org_name(org, root) bucket_obj = await root.client.buckets.get( - bucket, cluster_name=cluster, bucket_owner=owner + bucket, + cluster_name=cluster, + org_name=org_name, + project_name=project, ) if bucket_obj.imported: bc = await root.client.buckets.request_tmp_credentials( - bucket, cluster_name=cluster, bucket_owner=owner + bucket, + cluster_name=bucket_obj.cluster_name, + org_name=bucket_obj.org_name, + project_name=bucket_obj.project_name, ) credentials = bc.credentials else: @@ -383,19 +429,32 @@ async def statbucket( help="Look on a specified cluster (the current cluster by default).", ) @option( - "--owner", - type=str, - help="Owner of bucket to assume for named bucket (the current user by default)", + "--org", + type=ORG, + help="Look on a specified org (the current org by default).", +) +@option( + "--project", + type=PROJECT, + help="Look on a specified project (the current project by default).", ) @argument("bucket", type=BUCKET) async def du( - root: Root, cluster: Optional[str], owner: Optional[str], bucket: str + root: Root, + cluster: Optional[str], + org: Optional[str], + project: Optional[str], + bucket: str, ) -> None: """ Get storage usage for BUCKET. """ + org_name = parse_org_name(org, root) bucket_obj = await root.client.buckets.get( - bucket, cluster_name=cluster, bucket_owner=owner + bucket, + cluster_name=cluster, + org_name=org_name, + project_name=project, ) bucket_str = bucket_obj.name or bucket_obj.id @@ -406,7 +465,10 @@ async def du( with root.status(base_str) as status: async with root.client.buckets.get_disk_usage( - bucket_obj.id, cluster_name=cluster, bucket_owner=bucket_obj.owner + bucket_obj.id, + cluster_name=bucket_obj.cluster_name, + org_name=bucket_obj.org_name, + project_name=bucket_obj.project_name, ) as usage_it: async for usage in usage_it: status.update( @@ -426,45 +488,47 @@ async def du( type=CLUSTER, help="Perform on a specified cluster (the current cluster by default).", ) +@option( + "--org", + type=ORG, + help="Perform on a specified org (the current org by default).", +) +@option( + "--project", + type=PROJECT, + help="Perform on a specified project (the current project by default).", +) @option( "-f", "--force", is_flag=True, help="Force removal of all blobs inside bucket", ) -@option( - "--owner", - type=str, - help="Owner of bucket to assume for named bucket (the current user by default)", -) @argument("buckets", type=BUCKET, nargs=-1, required=True) async def rmbucket( root: Root, cluster: Optional[str], + org: Optional[str], + project: Optional[str], force: bool, - owner: Optional[str], buckets: Sequence[str], ) -> None: """ Remove bucket BUCKET. """ + org_name = parse_org_name(org, root) for bucket in buckets: bucket_id = await resolve_bucket( bucket, client=root.client, cluster_name=cluster, - bucket_owner=owner, + org_name=org_name, + project_name=project, ) if force: - bucket_obj = await root.client.buckets.get( - bucket_id, - cluster_name=cluster, - bucket_owner=owner, - ) + bucket_obj = await root.client.buckets.get(bucket_id, cluster_name=cluster) await root.client.buckets.blob_rm(bucket_obj.uri, recursive=True) - await root.client.buckets.rm( - bucket_id, cluster_name=cluster, bucket_owner=owner - ) + await root.client.buckets.rm(bucket_id, cluster) if root.verbosity >= 0: root.print(f"Bucket with id '{bucket_id}' was successfully removed.") @@ -476,9 +540,14 @@ async def rmbucket( help="Perform on a specified cluster (the current cluster by default).", ) @option( - "--owner", - type=str, - help="Owner of bucket to assume for named bucket (the current user by default)", + "--org", + type=ORG, + help="Perform on a specified org (the current org by default).", +) +@option( + "--project", + type=PROJECT, + help="Perform on a specified project (the current project by default).", ) @argument("bucket", type=BUCKET, required=True) @argument( @@ -489,7 +558,8 @@ async def rmbucket( async def set_bucket_publicity( root: Root, cluster: Optional[str], - owner: Optional[str], + org: Optional[str], + project: Optional[str], bucket: str, public_level: str, ) -> None: @@ -502,8 +572,13 @@ async def set_bucket_publicity( neuro blob set-bucket-publicity my-bucket private """ public = public_level == "public" + org_name = parse_org_name(org, root) await root.client.buckets.set_public_access( - bucket, public, cluster_name=cluster, bucket_owner=owner + bucket, + public, + cluster_name=cluster, + org_name=org_name, + project_name=project, ) if root.verbosity >= 0: root.print( @@ -611,7 +686,6 @@ async def glob(root: Root, full_uri: bool, patterns: Sequence[URL]) -> None: org_name=root.client.config.org_name, ) for pattern in patterns: - if root.verbosity > 0: painter = get_painter(root.color) uri_text = painter.paint(str(pattern), FileStatusType.FILE) @@ -1024,6 +1098,16 @@ async def lscredentials(root: Root, cluster: Optional[str]) -> None: type=CLUSTER, help="Perform in a specified cluster (the current cluster by default).", ) +@option( + "--org", + type=ORG, + help="Perform in a specified org (the current org by default).", +) +@option( + "--project", + type=PROJECT, + help="Perform in a specified project (the current project by default).", +) @option( "--name", type=str, @@ -1042,13 +1126,22 @@ async def mkcredentials( buckets: Sequence[str], name: Optional[str] = None, cluster: Optional[str] = None, + org: Optional[str] = None, + project_name: Optional[str] = None, read_only: bool = False, ) -> None: """ Create a new bucket credential. """ + org_name = parse_org_name(org, root) bucket_ids = [ - await resolve_bucket(bucket, client=root.client, cluster_name=cluster) + await resolve_bucket( + bucket, + client=root.client, + cluster_name=cluster, + org_name=org_name, + project_name=project_name, + ) for bucket in buckets ] credential = await root.client.buckets.persistent_credentials_create( diff --git a/neuro-cli/src/neuro_cli/click_types.py b/neuro-cli/src/neuro_cli/click_types.py index dca1a4689..067d54c8c 100644 --- a/neuro-cli/src/neuro_cli/click_types.py +++ b/neuro-cli/src/neuro_cli/click_types.py @@ -880,19 +880,27 @@ async def get_completions( incomplete: str, ) -> AsyncIterator[CompletionItem]: full_uri = root.client.parse.normalize_uri(uri) - bucket_id_complete = len(full_uri.parts) > 3 or ( - len(full_uri.parts) == 3 and incomplete.endswith("/") - ) - if not bucket_id_complete: + if not self._is_bucket_uri_complete(full_uri, root, incomplete): + prefix = uri.parent + full_prefix = full_uri.parent if uri.path else full_uri + full_prefix_str = str(full_prefix / "") + full_uri_str = str(full_uri if uri.path else full_uri / "") + completions = set() async with root.client.buckets.list(cluster_name=full_uri.host) as it: async for bucket in it: - if uri.host: - prefix = URL(f"blob://{full_uri.host}/{bucket.owner}") - else: - prefix = URL(f"blob:") - names = [bucket.id] + ([bucket.name] if bucket.name else []) - for name in names: - if str(prefix / name).startswith(incomplete): + bucket_uris = [bucket.uri] + if bucket.name: + bucket_uris = [bucket.uri.parent / bucket.id] + bucket_uris + for bucket_uri in bucket_uris: + bucket_uri_str = str(bucket_uri) + if not bucket_uri_str.startswith(full_uri_str): + continue + path_parts = bucket_uri_str[len(full_prefix_str) :].split("/") + if len(path_parts) == 0: + continue + name = path_parts[0] + if name not in completions: + completions.add(name) yield self._make_item(prefix, name, True) else: # Generic get_completions() is not used here because we can @@ -919,6 +927,20 @@ async def get_completions( yield self._make_item(prefix, item.name, item.is_dir()) + def _is_bucket_uri_complete(self, uri: URL, root: Root, incomplete: str) -> bool: + parts = uri.parts + if len(parts) > 4: + # Check uri has format blob://cluster/org/project/bucket/ + return True + if len(parts) == 4 or ( + len(parts) == 3 and parts[-1] and incomplete.endswith("/") + ): + assert uri.host + cluster = root.client.config.clusters.get(uri.host) + # Check uri has format blob://cluster/project/bucket/ + return bool(cluster and parts[1] not in cluster.orgs) + return False + class PlatformURIType(AsyncType[URL]): name = "uri" @@ -1226,3 +1248,4 @@ async def async_shell_complete( def setup_shell_completion() -> None: add_completion_class(NewZshComplete) add_completion_class(NewBashComplete) + add_completion_class(NewBashComplete) diff --git a/neuro-cli/src/neuro_cli/formatters/buckets.py b/neuro-cli/src/neuro_cli/formatters/buckets.py index 52a31d7b8..3031761a2 100644 --- a/neuro-cli/src/neuro_cli/formatters/buckets.py +++ b/neuro-cli/src/neuro_cli/formatters/buckets.py @@ -47,6 +47,7 @@ def _bucket_to_table_row(self, bucket: Bucket) -> Sequence[str]: if self._long_format: line += [ bucket.org_name or ORG.NO_ORG_STR, + bucket.project_name, self._datetime_formatter(bucket.created_at), "√" if bucket.public else "×", ] @@ -63,6 +64,7 @@ def __call__(self, buckets: Sequence[Bucket]) -> RenderableType: table.add_column("Uri") if self._long_format: table.add_column("Org name") + table.add_column("Project name") table.add_column("Created at") table.add_column("Public") for bucket in buckets: @@ -92,6 +94,7 @@ def __call__( if bucket.name: table.add_row("Name", bucket.name) table.add_row("Org name", bucket.org_name or ORG.NO_ORG_STR) + table.add_row("Project name", bucket.project_name) table.add_row("Created at", self._datetime_formatter(bucket.created_at)) table.add_row("Provider", bucket.provider) table.add_row("Imported", str(bucket.imported)) diff --git a/neuro-cli/src/neuro_cli/utils.py b/neuro-cli/src/neuro_cli/utils.py index 128baeb62..5619dd068 100644 --- a/neuro-cli/src/neuro_cli/utils.py +++ b/neuro-cli/src/neuro_cli/utils.py @@ -537,7 +537,8 @@ async def resolve_bucket( *, client: Client, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> str: # Temporary fast path. if re.fullmatch(BUCKET_ID_PATTERN, id_or_name): @@ -546,7 +547,8 @@ async def resolve_bucket( bucket = await client.buckets.get( id_or_name, cluster_name=cluster_name, - bucket_owner=bucket_owner, + org_name=org_name, + project_name=project_name, ) return bucket.id diff --git a/neuro-cli/tests/e2e/conftest.py b/neuro-cli/tests/e2e/conftest.py index ca869234f..cf990ca98 100644 --- a/neuro-cli/tests/e2e/conftest.py +++ b/neuro-cli/tests/e2e/conftest.py @@ -704,7 +704,12 @@ async def acreate_bucket(self, name: str, *, wait: bool = False) -> Bucket: async def adelete_bucket(self, bucket: Bucket) -> None: __tracebackhide__ = True async with api_get(timeout=CLIENT_TIMEOUT, path=self._nmrc_path) as client: - await client.buckets.rm(bucket.id, bucket_owner=bucket.owner) + await client.buckets.rm( + bucket.id, + cluster_name=bucket.cluster_name, + org_name=bucket.org_name, + project_name=bucket.project_name, + ) delete_bucket = run_async(adelete_bucket) @@ -724,7 +729,11 @@ async def acleanup_bucket(self, bucket: Bucket) -> None: log.info("Removing %s", blob.uri) tasks.append( client.buckets.delete_blob( - bucket.id, key=blob.key, bucket_owner=bucket.owner + bucket.id, + key=blob.key, + cluster_name=bucket.cluster_name, + org_name=bucket.org_name, + project_name=bucket.project_name, ) ) await asyncio.gather(*tasks) diff --git a/neuro-cli/tests/unit/conftest.py b/neuro-cli/tests/unit/conftest.py index 1ff955892..eee42aaf2 100644 --- a/neuro-cli/tests/unit/conftest.py +++ b/neuro-cli/tests/unit/conftest.py @@ -59,7 +59,7 @@ def nmrc_path(tmp_path: Path, token: str, auth_config: _AuthConfig) -> Path: ), }, name="default", - orgs=[None], + orgs=[None, "org"], ) cluster2_config = Cluster( registry_url=URL("https://registry2-dev.neu.ro"), diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_0.ref index d875a4511..51a82f325 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_0.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_0.ref @@ -1 +1 @@ -blob://test-cluster/test-user/neuro-my-bucket +blob://test-cluster/test-project/neuro-my-bucket diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_1.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_1.ref index fd8221ef5..eb8802984 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_1.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_1.ref @@ -1 +1 @@ -blob://test-cluster/public/neuro-public-bucket +blob://test-cluster/test-project/neuro-public-bucket diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_2.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_2.ref index bca85ce8f..d99b81c95 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_2.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_2.ref @@ -1 +1 @@ -blob://test-cluster/test-org/another-user/neuro-shared-bucket +blob://test-cluster/test-org/test-project/neuro-shared-bucket diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_3.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_3.ref index 6c2166589..d8dd84cff 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_3.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_3.ref @@ -1 +1 @@ -blob://test-cluster/test-user/neuro-my-bucket/file1024.txt +blob://test-cluster/test-project/neuro-my-bucket/file1024.txt diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_4.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_4.ref index 9b1dad38e..db6b5f8aa 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_4.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_4.ref @@ -1 +1 @@ -blob://test-cluster/public/neuro-public-bucket/file_bigger.txt +blob://test-cluster/test-project/neuro-public-bucket/file_bigger.txt diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_5.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_5.ref index 0b118350a..705cffca1 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_5.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_5.ref @@ -1 +1 @@ -blob://test-cluster/test-org/another-user/neuro-shared-bucket/folder2/info.txt +blob://test-cluster/test-org/test-project/neuro-shared-bucket/folder2/info.txt diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_6.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_6.ref index 9cf5baef9..2a0625ee5 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_6.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_6.ref @@ -1 +1 @@ -blob://test-cluster/test-org/another-user/neuro-shared-bucket/folder2/ +blob://test-cluster/test-org/test-project/neuro-shared-bucket/folder2/ diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_7.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_7.ref index cbf80a9db..19b469a19 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_7.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_7.ref @@ -1 +1 @@ -blob://test-cluster/test-user/neuro-my-bucket/folder1/ +blob://test-cluster/test-project/neuro-my-bucket/folder1/ diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_8.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_8.ref index 90be94ae9..b6bf6e458 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_8.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter0]_8.ref @@ -1 +1 @@ -blob://test-cluster/public/neuro-public-bucket/folder2/ +blob://test-cluster/test-project/neuro-public-bucket/folder2/ diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_0.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_0.ref index d16843a54..339112a71 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_0.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_0.ref @@ -1 +1 @@ -bucket 2018-01-01 03:00:00 blob://test-cluster/test-user/neuro-my-bucket +bucket 2018-01-01 03:00:00 blob://test-cluster/test-project/neuro-my-bucket diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_1.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_1.ref index 8d519c5fb..c1126c768 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_1.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_1.ref @@ -1 +1 @@ -bucket 2018-01-01 17:02:04 blob://test-cluster/public/neuro-public-bucket +bucket 2018-01-01 17:02:04 blob://test-cluster/test-project/neuro-public-bucket diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_2.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_2.ref index 84313acee..737a381a2 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_2.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_2.ref @@ -1 +1 @@ -bucket 2018-01-01 13:01:05 blob://test-cluster/test-org/another-user/neuro-shared-bucket +bucket 2018-01-01 13:01:05 blob://test-cluster/test-org/test-project/neuro-shared-bucket diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_3.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_3.ref index 6bc3cd21d..83b02b6b1 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_3.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_3.ref @@ -1 +1 @@ -obj 1024 2018-01-01 14:00:00 blob://test-cluster/test-user/neuro-my-bucket/file1024.txt +obj 1024 2018-01-01 14:00:00 blob://test-cluster/test-project/neuro-my-bucket/file1024.txt diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_4.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_4.ref index a28a839ee..fc9fc3727 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_4.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_4.ref @@ -1 +1 @@ -obj 1024001 2018-01-01 14:00:00 blob://test-cluster/public/neuro-public-bucket/file_bigger.txt +obj 1024001 2018-01-01 14:00:00 blob://test-cluster/test-project/neuro-public-bucket/file_bigger.txt diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_5.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_5.ref index e886f2417..d1ed8ecdf 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_5.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_5.ref @@ -1 +1 @@ -obj 240 2018-01-01 14:00:00 blob://test-cluster/test-org/another-user/neuro-shared-bucket/folder2/info.txt +obj 240 2018-01-01 14:00:00 blob://test-cluster/test-org/test-project/neuro-shared-bucket/folder2/info.txt diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_6.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_6.ref index 44379a1c7..5fde9001d 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_6.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_6.ref @@ -1 +1 @@ -dir 0 2018-01-01 14:00:00 blob://test-cluster/test-org/another-user/neuro-shared-bucket/folder2/ +dir 0 2018-01-01 14:00:00 blob://test-cluster/test-org/test-project/neuro-shared-bucket/folder2/ diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_7.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_7.ref index 3dbb94c04..918192c93 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_7.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_7.ref @@ -1 +1 @@ -dir 0 blob://test-cluster/test-user/neuro-my-bucket/folder1/ +dir 0 blob://test-cluster/test-project/neuro-my-bucket/folder1/ diff --git a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_8.ref b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_8.ref index c3472a33d..596c9bf08 100644 --- a/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_8.ref +++ b/neuro-cli/tests/unit/formatters/ascii/TestBlobFormatter.test_long_formatter[formatter1]_8.ref @@ -1 +1 @@ -dir 0 blob://test-cluster/public/neuro-public-bucket/folder2/ +dir 0 blob://test-cluster/test-project/neuro-public-bucket/folder2/ diff --git a/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_0.ref b/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_0.ref index eb66080c3..68e67e1e0 100644 --- a/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_0.ref +++ b/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_0.ref @@ -1,8 +1,9 @@ -Id bucket - Uri blob://cluster/user/test-bucket - Name test-bucket - Org name NO_ORG - Created at Mar 04 2017 - Provider aws - Imported False - Public False +Id bucket + Uri blob://cluster/test-project/test-bucket + Name test-bucket + Org name NO_ORG + Project name test-project + Created at Mar 04 2017 + Provider aws + Imported False + Public False diff --git a/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_with_org_0.ref b/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_with_org_0.ref index 76a73ea0e..990b8f024 100644 --- a/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_with_org_0.ref +++ b/neuro-cli/tests/unit/formatters/ascii/test_bucket_formatter_with_org_0.ref @@ -1,8 +1,9 @@ -Id bucket - Uri blob://cluster/test-org/user/test-bucket - Name test-bucket - Org name test-org - Created at Mar 04 2017 - Provider aws - Imported False - Public False +Id bucket + Uri blob://cluster/test-org/test-project/test-bucket + Name test-bucket + Org name test-org + Project name test-project + Created at Mar 04 2017 + Provider aws + Imported False + Public False diff --git a/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_long_0.ref b/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_long_0.ref index d08cc8ebf..b60b163cf 100644 --- a/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_long_0.ref +++ b/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_long_0.ref @@ -1,6 +1,6 @@ -Id Name Provider Uri Org name Created at Public - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - bucket-1 test-bucket aws blob://cluster/user/test-bucket NO_ORG Mar 04 2017 × - bucket-2 test-bucket-2 aws blob://cluster/user/test-bucket-2 NO_ORG Mar 04 2016 × - bucket-3 aws blob://cluster/test-org/user-2/bucket-3 test-org Mar 04 2018 × - bucket-4 aws blob://cluster/test-org/user/bucket-4 test-org Mar 04 2019 √ +Id Name Provider Uri Org name Project name Created at Public + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + bucket-1 test-bucket aws blob://cluster/test-project/test-… NO_ORG test-project Mar 04 2017 × + bucket-2 test-bucket-2 aws blob://cluster/test-project/test-… NO_ORG test-project Mar 04 2016 × + bucket-3 aws blob://cluster/test-org/test-proj… test-org test-project Mar 04 2018 × + bucket-4 aws blob://cluster/test-org/test-proj… test-org test-project Mar 04 2019 √ diff --git a/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_short_0.ref b/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_short_0.ref index 01977eb38..52d25f216 100644 --- a/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_short_0.ref +++ b/neuro-cli/tests/unit/formatters/ascii/test_buckets_formatter_short_0.ref @@ -1,6 +1,6 @@ -Id Name Provider Uri - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - bucket-1 test-bucket aws blob://cluster/user/test-bucket - bucket-2 test-bucket-2 aws blob://cluster/user/test-bucket-2 - bucket-3 aws blob://cluster/test-org/user-2/bucket-3 - bucket-4 aws blob://cluster/test-org/user/bucket-4 +Id Name Provider Uri + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + bucket-1 test-bucket aws blob://cluster/test-project/test-bucket + bucket-2 test-bucket-2 aws blob://cluster/test-project/test-bucket-2 + bucket-3 aws blob://cluster/test-org/test-project/bucket-3 + bucket-4 aws blob://cluster/test-org/test-project/bucket-4 diff --git a/neuro-cli/tests/unit/formatters/test_blob_formatters.py b/neuro-cli/tests/unit/formatters/test_blob_formatters.py index f3073a7be..4064bcb3b 100644 --- a/neuro-cli/tests/unit/formatters/test_blob_formatters.py +++ b/neuro-cli/tests/unit/formatters/test_blob_formatters.py @@ -13,7 +13,6 @@ class TestBlobFormatter: - buckets: List[Bucket] = [ Bucket( id="bucket-1", @@ -24,6 +23,7 @@ class TestBlobFormatter: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-2", @@ -34,6 +34,7 @@ class TestBlobFormatter: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-3", @@ -44,6 +45,7 @@ class TestBlobFormatter: provider=Bucket.Provider.AWS, imported=False, org_name="test-org", + project_name="test-project", ), ] diff --git a/neuro-cli/tests/unit/formatters/test_bucket_credentials.py b/neuro-cli/tests/unit/formatters/test_bucket_credentials.py index a28cf8ea8..9c62ca452 100644 --- a/neuro-cli/tests/unit/formatters/test_bucket_credentials.py +++ b/neuro-cli/tests/unit/formatters/test_bucket_credentials.py @@ -22,6 +22,7 @@ async def test_bucket_credentials_formatter(rich_cmp: Any) -> None: created_at=isoparse("2017-03-04T12:28:59.759433+00:00"), imported=False, org_name=None, + project_name="test-project", ) credentials = PersistentBucketCredentials( id="bucket-credentials", @@ -66,6 +67,7 @@ def credentials_list_fixture() -> CredListFixture: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-2", @@ -76,6 +78,7 @@ def credentials_list_fixture() -> CredListFixture: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-3", @@ -86,6 +89,7 @@ def credentials_list_fixture() -> CredListFixture: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-4", @@ -96,6 +100,7 @@ def credentials_list_fixture() -> CredListFixture: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), ] diff --git a/neuro-cli/tests/unit/formatters/test_buckets.py b/neuro-cli/tests/unit/formatters/test_buckets.py index 25ffdb691..4fe7d65df 100644 --- a/neuro-cli/tests/unit/formatters/test_buckets.py +++ b/neuro-cli/tests/unit/formatters/test_buckets.py @@ -23,6 +23,7 @@ def test_bucket_formatter(rich_cmp: Any) -> None: created_at=isoparse("2017-03-04T12:28:59.759433+00:00"), imported=False, org_name=None, + project_name="test-project", ) fmtr = BucketFormatter(str, datetime_formatter=format_datetime_human) rich_cmp(fmtr(bucket)) @@ -38,6 +39,7 @@ def test_bucket_formatter_with_org(rich_cmp: Any) -> None: created_at=isoparse("2017-03-04T12:28:59.759433+00:00"), imported=False, org_name="test-org", + project_name="test-project", ) fmtr = BucketFormatter(str, datetime_formatter=format_datetime_human) rich_cmp(fmtr(bucket)) @@ -55,6 +57,7 @@ def buckets_list() -> List[Bucket]: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-2", @@ -65,6 +68,7 @@ def buckets_list() -> List[Bucket]: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-3", @@ -75,6 +79,7 @@ def buckets_list() -> List[Bucket]: provider=Bucket.Provider.AWS, imported=False, org_name="test-org", + project_name="test-project", ), Bucket( id="bucket-4", @@ -86,6 +91,7 @@ def buckets_list() -> List[Bucket]: imported=False, public=True, org_name="test-org", + project_name="test-project", ), ] diff --git a/neuro-cli/tests/unit/test_shell_completion.py b/neuro-cli/tests/unit/test_shell_completion.py index c23c86547..c4724adca 100644 --- a/neuro-cli/tests/unit/test_shell_completion.py +++ b/neuro-cli/tests/unit/test_shell_completion.py @@ -326,6 +326,7 @@ async def list(cluster_name: str) -> AsyncIterator[Bucket]: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="project", ) yield Bucket( id="bucket-2", @@ -336,6 +337,7 @@ async def list(cluster_name: str) -> AsyncIterator[Bucket]: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="project", ) yield Bucket( id="bucket-3", @@ -346,6 +348,40 @@ async def list(cluster_name: str) -> AsyncIterator[Bucket]: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="project", + ) + yield Bucket( + id="bucket-4", + name="neuro-my-org-bucket", + created_at=datetime(2018, 1, 1, 3), + cluster_name="default", + owner="user", + provider=Bucket.Provider.AWS, + imported=False, + org_name="org", + project_name="project", + ) + yield Bucket( + id="bucket-5", + name="neuro-public-org-bucket", + created_at=datetime(2018, 1, 1, 17, 2, 4), + cluster_name="default", + owner="public", + provider=Bucket.Provider.AWS, + imported=False, + org_name="org", + project_name="project", + ) + yield Bucket( + id="bucket-6", + name="neuro-shared-org-bucket", + created_at=datetime(2018, 1, 1, 13, 1, 5), + cluster_name="default", + owner="another-user", + provider=Bucket.Provider.AWS, + imported=False, + org_name="org", + project_name="project", ) async def blob_is_dir(uri: URL) -> bool: @@ -454,6 +490,45 @@ async def list_blobs(uri: URL) -> AsyncIterator[BlobObject]: assert bash_out == "uri,bucket-1/," assert zsh_out == "uri\nbucket-1/\n_\nblob:" + zsh_out, bash_out = run_autocomplete(["blob", "ls", "blob:/p"]) + assert bash_out == "uri,project/,//default/" + assert zsh_out == "uri\nproject/\n_\nblob://default/" + + zsh_out, bash_out = run_autocomplete(["blob", "ls", "blob:/project/"]) + assert bash_out == "uri,project/,//default/" + assert zsh_out == "uri\nproject/\n_\nblob://default/" + + zsh_out, bash_out = run_autocomplete(["blob", "ls", "blob://default/"]) + assert bash_out == "uri,project/,//default/\nuri,org/,//default/" + assert ( + zsh_out + == "uri\nproject/\n_\nblob://default/\nuri\norg/\n_\nblob://default/" + ) + + zsh_out, bash_out = run_autocomplete(["blob", "ls", "blob://default/org/"]) + assert bash_out == "uri,project/,//default/org/" + assert zsh_out == "uri\nproject/\n_\nblob://default/org/" + + zsh_out, bash_out = run_autocomplete( + ["blob", "ls", "blob://default/org/project/"] + ) + assert bash_out == ( + "uri,bucket-4/,//default/org/project/\n" + "uri,neuro-my-org-bucket/,//default/org/project/\n" + "uri,bucket-5/,//default/org/project/\n" + "uri,neuro-public-org-bucket/,//default/org/project/\n" + "uri,bucket-6/,//default/org/project/\n" + "uri,neuro-shared-org-bucket/,//default/org/project/" + ) + assert zsh_out == ( + "uri\nbucket-4/\n_\nblob://default/org/project/\n" + "uri\nneuro-my-org-bucket/\n_\nblob://default/org/project/\n" + "uri\nbucket-5/\n_\nblob://default/org/project/\n" + "uri\nneuro-public-org-bucket/\n_\nblob://default/org/project/\n" + "uri\nbucket-6/\n_\nblob://default/org/project/\n" + "uri\nneuro-shared-org-bucket/\n_\nblob://default/org/project/" + ) + zsh_out, bash_out = run_autocomplete(["blob", "ls", "blob:bucket-1/"]) assert bash_out == ( "uri,file1024.txt,bucket-1/\n" @@ -1191,6 +1266,7 @@ def test_bucket_autocomplete(run_autocomplete: _RunAC) -> None: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-2", @@ -1201,6 +1277,7 @@ def test_bucket_autocomplete(run_autocomplete: _RunAC) -> None: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-3", @@ -1211,6 +1288,7 @@ def test_bucket_autocomplete(run_autocomplete: _RunAC) -> None: provider=Bucket.Provider.AWS, imported=False, org_name="test-org", + project_name="test-project", ), Bucket( id="bucket-4", @@ -1222,6 +1300,7 @@ def test_bucket_autocomplete(run_autocomplete: _RunAC) -> None: imported=False, public=True, org_name="test-org", + project_name="test-project", ), ], "other": [ @@ -1234,6 +1313,7 @@ def test_bucket_autocomplete(run_autocomplete: _RunAC) -> None: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), ], } diff --git a/neuro-sdk/src/neuro_sdk/_bucket_base.py b/neuro-sdk/src/neuro_sdk/_bucket_base.py index da0a900cd..6c22de9e7 100644 --- a/neuro-sdk/src/neuro_sdk/_bucket_base.py +++ b/neuro-sdk/src/neuro_sdk/_bucket_base.py @@ -136,6 +136,7 @@ class Bucket: owner: str cluster_name: str org_name: Optional[str] + project_name: str provider: "Bucket.Provider" created_at: datetime imported: bool @@ -147,7 +148,7 @@ def uri(self) -> URL: base = f"blob://{self.cluster_name}" if self.org_name: base += f"/{self.org_name}" - return URL(f"{base}/{self.owner}/{self.name or self.id}") + return URL(f"{base}/{self.project_name}/{self.name or self.id}") def get_key_for_uri(self, uri: URL) -> str: self_uris = [self.uri] diff --git a/neuro-sdk/src/neuro_sdk/_buckets.py b/neuro-sdk/src/neuro_sdk/_buckets.py index 14707fc24..a023d92c8 100644 --- a/neuro-sdk/src/neuro_sdk/_buckets.py +++ b/neuro-sdk/src/neuro_sdk/_buckets.py @@ -196,6 +196,7 @@ def _parse_bucket_payload(self, payload: Mapping[str, Any]) -> Bucket: public=payload.get("public", False), cluster_name=self._config.cluster_name, org_name=payload.get("org_name"), + project_name=payload["project_name"], ) def _parse_bucket_credentials_payload( @@ -212,12 +213,33 @@ def _get_buckets_url(self, cluster_name: Optional[str]) -> URL: cluster_name = self._config.cluster_name return self._config.get_cluster(cluster_name).buckets_url / "buckets" + def _get_bucket_url_params( + self, org_name: Optional[str], project_name: Optional[str] + ) -> Dict[str, str]: + params = { + "org_name": org_name or self._config.org_name or "NO_ORG", + "project_name": project_name or self._config.project_name_or_raise, + } + return params + @asyncgeneratorcontextmanager - async def list(self, cluster_name: Optional[str] = None) -> AsyncIterator[Bucket]: + async def list( + self, + cluster_name: Optional[str] = None, + org_name: Union[Optional[str], OrgNameSentinel] = ORG_NAME_SENTINEL, + project_name: Optional[str] = None, + ) -> AsyncIterator[Bucket]: url = self._get_buckets_url(cluster_name) auth = await self._config._api_auth() headers = {"Accept": "application/x-ndjson"} - async with self._core.request("GET", url, headers=headers, auth=auth) as resp: + params = {} + if not isinstance(org_name, OrgNameSentinel): + params["org_name"] = org_name or "NO_ORG" + if project_name: + params["project_name"] = project_name + async with self._core.request( + "GET", url, headers=headers, auth=auth, params=params + ) as resp: if resp.headers.get("Content-Type", "").startswith("application/x-ndjson"): async for line in resp.content: server_message = json.loads(line) @@ -234,6 +256,7 @@ async def create( name: Optional[str] = None, cluster_name: Optional[str] = None, org_name: Union[Optional[str], OrgNameSentinel] = ORG_NAME_SENTINEL, + project_name: Optional[str] = None, ) -> Bucket: url = self._get_buckets_url(cluster_name) auth = await self._config._api_auth() @@ -242,6 +265,7 @@ async def create( "org_name": org_name if not isinstance(org_name, OrgNameSentinel) else self._config.org_name, + "project_name": project_name or self._config.project_name_or_raise, } async with self._core.request("POST", url, auth=auth, json=data) as resp: payload = await resp.json() @@ -255,6 +279,7 @@ async def import_external( name: Optional[str] = None, cluster_name: Optional[str] = None, org_name: Union[Optional[str], OrgNameSentinel] = ORG_NAME_SENTINEL, + project_name: Optional[str] = None, ) -> Bucket: url = self._get_buckets_url(cluster_name) / "import" / "external" auth = await self._config._api_auth() @@ -266,6 +291,7 @@ async def import_external( "org_name": org_name if not isinstance(org_name, OrgNameSentinel) else self._config.org_name, + "project_name": project_name or self._config.project_name_or_raise, } async with self._core.request("POST", url, auth=auth, json=data) as resp: payload = await resp.json() @@ -275,12 +301,13 @@ async def get( self, bucket_id_or_name: str, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> Bucket: url = self._get_buckets_url(cluster_name) / bucket_id_or_name - query = {"owner": bucket_owner} if bucket_owner else {} + params = self._get_bucket_url_params(org_name, project_name) auth = await self._config._api_auth() - async with self._core.request("GET", url, auth=auth, params=query) as resp: + async with self._core.request("GET", url, auth=auth, params=params) as resp: payload = await resp.json() return self._parse_bucket_payload(payload) @@ -288,12 +315,13 @@ async def rm( self, bucket_id_or_name: str, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> None: url = self._get_buckets_url(cluster_name) / bucket_id_or_name - query = {"owner": bucket_owner} if bucket_owner else {} + params = self._get_bucket_url_params(org_name, project_name) auth = await self._config._api_auth() - async with self._core.request("DELETE", url, auth=auth, params=query): + async with self._core.request("DELETE", url, auth=auth, params=params): pass async def set_public_access( @@ -301,16 +329,17 @@ async def set_public_access( bucket_id_or_name: str, public_access: bool, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> Bucket: url = self._get_buckets_url(cluster_name) / bucket_id_or_name + params = self._get_bucket_url_params(org_name, project_name) auth = await self._config._api_auth() data = { "public": public_access, } - query = {"owner": bucket_owner} if bucket_owner else {} async with self._core.request( - "PATCH", url, auth=auth, json=data, params=query + "PATCH", url, auth=auth, json=data, params=params ) as resp: payload = await resp.json() return self._parse_bucket_payload(payload) @@ -319,16 +348,17 @@ async def request_tmp_credentials( self, bucket_id_or_name: str, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> BucketCredentials: url = ( self._get_buckets_url(cluster_name) / bucket_id_or_name / "make_tmp_credentials" ) + params = self._get_bucket_url_params(org_name, project_name) auth = await self._config._api_auth() - query = {"owner": bucket_owner} if bucket_owner else {} - async with self._core.request("POST", url, auth=auth, params=query) as resp: + async with self._core.request("POST", url, auth=auth, params=params) as resp: payload = await resp.json() return self._parse_bucket_credentials_payload(payload) @@ -337,12 +367,16 @@ async def get_disk_usage( self, bucket_id_or_name: str, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> AsyncIterator[BucketUsage]: total_bytes = 0 obj_count = 0 async with self._get_provider_by_exact( - bucket_id_or_name, cluster_name, bucket_owner + bucket_id_or_name, + cluster_name=cluster_name, + org_name=org_name, + project_name=project_name, ) as provider: async with provider.list_blobs("", recursive=True) as it: async for obj in it: @@ -373,10 +407,14 @@ async def _get_provider_by_exact( self, bucket_id_or_name: str, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> AsyncIterator[BucketProvider]: bucket = await self.get( - bucket_id_or_name, cluster_name=cluster_name, bucket_owner=bucket_owner + bucket_id_or_name, + cluster_name=cluster_name, + org_name=org_name, + project_name=project_name, ) async with self._get_provider_for_bucket(bucket) as provider: yield provider @@ -424,10 +462,14 @@ async def head_blob( bucket_id_or_name: str, key: str, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> BucketEntry: async with self._get_provider_by_exact( - bucket_id_or_name, cluster_name, bucket_owner + bucket_id_or_name, + cluster_name=cluster_name, + org_name=org_name, + project_name=project_name, ) as provider: return await provider.head_blob(key) @@ -437,10 +479,14 @@ async def put_blob( key: str, body: Union[AsyncIterator[bytes], bytes], cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> None: async with self._get_provider_by_exact( - bucket_id_or_name, cluster_name, bucket_owner + bucket_id_or_name, + cluster_name=cluster_name, + org_name=org_name, + project_name=project_name, ) as provider: await provider.put_blob(key, body) @@ -451,10 +497,14 @@ async def fetch_blob( key: str, offset: int = 0, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> AsyncIterator[bytes]: async with self._get_provider_by_exact( - bucket_id_or_name, cluster_name, bucket_owner + bucket_id_or_name, + cluster_name=cluster_name, + org_name=org_name, + project_name=project_name, ) as provider: async with provider.fetch_blob(key, offset=offset) as it: async for chunk in it: @@ -465,10 +515,14 @@ async def delete_blob( bucket_id_or_name: str, key: str, cluster_name: Optional[str] = None, - bucket_owner: Optional[str] = None, + org_name: Optional[str] = None, + project_name: Optional[str] = None, ) -> None: async with self._get_provider_by_exact( - bucket_id_or_name, cluster_name, bucket_owner + bucket_id_or_name, + cluster_name=cluster_name, + org_name=org_name, + project_name=project_name, ) as provider: return await provider.delete_blob(key) @@ -585,7 +639,6 @@ async def download_file( src = self._parser.normalize_uri(src, allowed_schemes=("blob",)) dst = normalize_local_path_uri(dst) async with self._get_bucket_fs(src) as bucket_fs: - src_key = bucket_fs.bucket.get_key_for_uri(src) transferer = FileTransferer(bucket_fs, LocalFS()) await transferer.transfer_file( @@ -609,7 +662,6 @@ async def upload_dir( src = normalize_local_path_uri(src) dst = self._parser.normalize_uri(dst, allowed_schemes=("blob",)) async with self._get_bucket_fs(dst) as bucket_fs: - dst_key = bucket_fs.bucket.get_key_for_uri(dst) transferer = FileTransferer(LocalFS(), bucket_fs) await transferer.transfer_dir( diff --git a/neuro-sdk/src/neuro_sdk/_parser.py b/neuro-sdk/src/neuro_sdk/_parser.py index 2e32edd83..276bc6382 100644 --- a/neuro-sdk/src/neuro_sdk/_parser.py +++ b/neuro-sdk/src/neuro_sdk/_parser.py @@ -305,7 +305,7 @@ def normalize_uri( _check_scheme(uri.scheme, allowed_schemes) ret = _normalize_uri( uri, - self._config.username, + self._config.project_name_or_raise, cluster_name or self._config.cluster_name, org_name if not isinstance(org_name, _Unset) else self._config.org_name, ) diff --git a/neuro-sdk/tests/test_blob_storage.py b/neuro-sdk/tests/test_blob_storage.py index 83267f891..8f834845d 100644 --- a/neuro-sdk/tests/test_blob_storage.py +++ b/neuro-sdk/tests/test_blob_storage.py @@ -108,6 +108,7 @@ def mock_bucket() -> Bucket: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ) diff --git a/neuro-sdk/tests/test_buckets.py b/neuro-sdk/tests/test_buckets.py index 84098147a..7e9169daa 100644 --- a/neuro-sdk/tests/test_buckets.py +++ b/neuro-sdk/tests/test_buckets.py @@ -28,6 +28,7 @@ async def handler(request: web.Request) -> web.Response: "created_at": created_at.isoformat(), "imported": False, "org_name": None, + "project_name": "test-project", }, { "id": "bucket-2", @@ -37,6 +38,7 @@ async def handler(request: web.Request) -> web.Response: "created_at": created_at.isoformat(), "imported": True, "org_name": "test-org", + "project_name": "test-project", }, ] ) @@ -63,6 +65,7 @@ async def handler(request: web.Request) -> web.Response: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ), Bucket( id="bucket-2", @@ -73,6 +76,7 @@ async def handler(request: web.Request) -> web.Response: provider=Bucket.Provider.AWS, imported=True, org_name="test-org", + project_name="test-project", ), ] @@ -89,6 +93,7 @@ async def handler(request: web.Request) -> web.Response: assert data == { "name": "test-bucket", "org_name": None, + "project_name": "test-project", } return web.json_response( { @@ -97,6 +102,7 @@ async def handler(request: web.Request) -> web.Response: "name": "test-bucket", "created_at": created_at.isoformat(), "provider": "aws", + "project_name": "test-project", } ) @@ -116,6 +122,7 @@ async def handler(request: web.Request) -> web.Response: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ) @@ -131,6 +138,7 @@ async def handler(request: web.Request) -> web.Response: assert data == { "name": "test-bucket", "org_name": "test-org", + "project_name": "test-project", } return web.json_response( { @@ -140,6 +148,7 @@ async def handler(request: web.Request) -> web.Response: "created_at": created_at.isoformat(), "provider": "aws", "org_name": "test-org", + "project_name": "test-project", } ) @@ -159,6 +168,7 @@ async def handler(request: web.Request) -> web.Response: provider=Bucket.Provider.AWS, imported=False, org_name="test-org", + project_name="test-project", ) @@ -177,6 +187,7 @@ async def handler(request: web.Request) -> web.Response: "provider_bucket_name": "test-external", "credentials": {"key": "value"}, "org_name": None, + "project_name": "test-project", } return web.json_response( { @@ -186,6 +197,7 @@ async def handler(request: web.Request) -> web.Response: "created_at": created_at.isoformat(), "provider": "aws", "imported": True, + "project_name": "test-project", } ) @@ -210,6 +222,7 @@ async def handler(request: web.Request) -> web.Response: provider=Bucket.Provider.AWS, imported=True, org_name=None, + project_name="test-project", ) @@ -228,6 +241,7 @@ async def handler(request: web.Request) -> web.Response: "provider_bucket_name": "test-external", "credentials": {"key": "value"}, "org_name": "test-org", + "project_name": "test-project", } return web.json_response( { @@ -238,6 +252,7 @@ async def handler(request: web.Request) -> web.Response: "provider": "aws", "imported": True, "org_name": "test-org", + "project_name": "test-project", } ) @@ -253,6 +268,7 @@ async def handler(request: web.Request) -> web.Response: credentials={"key": "value"}, name="test-bucket", org_name="test-org", + project_name="test-project", ) assert bucket == Bucket( id="bucket-1", @@ -263,6 +279,7 @@ async def handler(request: web.Request) -> web.Response: provider=Bucket.Provider.AWS, imported=True, org_name="test-org", + project_name="test-project", ) @@ -280,6 +297,7 @@ async def handler(request: web.Request) -> web.Response: "id": "bucket-1", "owner": "user", "name": "name", + "project_name": "test-project", "provider": "aws", "created_at": created_at.isoformat(), } @@ -301,6 +319,7 @@ async def handler(request: web.Request) -> web.Response: provider=Bucket.Provider.AWS, imported=False, org_name=None, + project_name="test-project", ) @@ -320,6 +339,7 @@ async def handler(request: web.Request) -> web.Response: "id": "bucket-1", "owner": "user", "name": "name", + "project_name": "test-project", "provider": "aws", "created_at": created_at.isoformat(), "public": True,