From 42117112c4d9f87a96d47b277c7d8ea0011a0619 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 17 Aug 2021 14:00:40 +0300 Subject: [PATCH] Implement public URI helpers (#2253) --- CHANGELOG.D/2253.feature | 1 + neuro-cli/tests/unit/test_utils.py | 10 ++-- neuro-sdk/docs/parse_reference.rst | 73 +++++++++++++++++++++++++++- neuro-sdk/src/neuro_sdk/parser.py | 66 ++++++++++++++++++++++++- neuro-sdk/src/neuro_sdk/url_utils.py | 28 ++++++++--- 5 files changed, 162 insertions(+), 16 deletions(-) create mode 100644 CHANGELOG.D/2253.feature diff --git a/CHANGELOG.D/2253.feature b/CHANGELOG.D/2253.feature new file mode 100644 index 000000000..fbdca67e9 --- /dev/null +++ b/CHANGELOG.D/2253.feature @@ -0,0 +1 @@ +Implement public URI helpers. diff --git a/neuro-cli/tests/unit/test_utils.py b/neuro-cli/tests/unit/test_utils.py index e5ff153a7..5928ee981 100644 --- a/neuro-cli/tests/unit/test_utils.py +++ b/neuro-cli/tests/unit/test_utils.py @@ -475,9 +475,9 @@ def test_parse_file_resource_no_scheme(root: Root) -> None: def test_parse_file_resource_unsupported_scheme(root: Root) -> None: - with pytest.raises(ValueError, match=r"Unsupported URI scheme"): + with pytest.raises(ValueError, match=r"Invalid scheme"): parse_file_resource("http://neu.ro", root) - with pytest.raises(ValueError, match=r"Unsupported URI scheme"): + with pytest.raises(ValueError, match=r"Invalid scheme"): parse_file_resource("image:ubuntu", root) @@ -554,11 +554,11 @@ def test_parse_resource_for_sharing_no_scheme(root: Root) -> None: def test_parse_resource_for_sharing_unsupported_scheme(root: Root) -> None: - with pytest.raises(ValueError, match=r"Unsupported URI scheme"): + with pytest.raises(ValueError, match=r"Invalid scheme"): parse_resource_for_sharing("http://neu.ro", root) - with pytest.raises(ValueError, match=r"Unsupported URI scheme"): + with pytest.raises(ValueError, match=r"Invalid scheme"): parse_resource_for_sharing("file:///etc/password", root) - with pytest.raises(ValueError, match=r"Unsupported URI scheme"): + with pytest.raises(ValueError, match=r"Invalid scheme"): parse_resource_for_sharing(r"c:scheme-less/resource", root) diff --git a/neuro-sdk/docs/parse_reference.rst b/neuro-sdk/docs/parse_reference.rst index 27b50b031..e21870b7d 100644 --- a/neuro-sdk/docs/parse_reference.rst +++ b/neuro-sdk/docs/parse_reference.rst @@ -69,8 +69,9 @@ Parser .. method:: volumes(volume: Sequence[str]) -> VolumeParseResult - Parse a sequence of volume definition into a tuple of three mappings - first one for - all regular volumes, second one for volumes using secrets and third for disk volumes. + Parse a sequence of volume definition into a tuple of three mappings - first one + for all regular volumes, second one for volumes using secrets and third for disk + volumes. :param ~typing.Sequence[str] env: Sequence of volumes specification. Each element can be either: @@ -81,6 +82,74 @@ Parser :return: :class:`VolumeParseResult` with parsing result + .. method:: uri_to_str(uri: URL) -> str + + Convert :class:`~yarl.URL` object into :class:`str`. + + .. method:: str_to_uri(uri: str, *, allowed_schemes: Iterable[str] = (), \ + cluster_name: Optional[str] = None) -> URL + + Parse a string into *normalized* :class:`URL` for future usage by SDK methods. + + :param str uri: an URI (``'storage:folder/file.txt'``) or local file path + (``'/home/user/folder/file.txt'``) to parse. + + :param ~typing.Iterable[str] allowed_schemes: an *iterable* of accepted URI + schemes, e.g. ``('file', + 'storage')``. No scheme check is + performed by default. + + :param ~typing.Optional[str] cluster_name: optional cluster name, the default + cluster is used if not specified. + + :return: :class:`~yarl.URL` with parsed URI. + + :raise ValueError: if ``uri`` is invalid or provides a scheme not enumerated by + ``allowed_schemes`` argument. + + .. method:: uri_to_path(uri: URL, *, cluster_name: Optional[str] = None) -> Path + + Convert :class:`~yarl.URL` into :class:`~pathlib.Path`. + + :raise ValueError: if ``uri`` has no ``'file:'`` scheme. + + .. method:: path_to_uri(path: Path) -> URL + + Convert :class:`~pathlib.Path` object into *normalized* :class:`~yarl.URL` with + ``'file:'`` scheme. + + :param ~pathlib.Path path: a path to convert. + + :param ~typing.Optional[str] cluster_name: optional cluster name, the default + cluster is used if not specified. + + :return: :class:`~yarl.URL` that represent a ``path``. + + .. method:: normalize_uri(uri: URL, *, allowed_schemes: Iterable[str] = (), \ + cluster_name: Optional[str] = None) -> URL + + Normalize ``uri`` according to current user name, cluster and allowed schemes. + + *Normalized* form is the minimal possible representation of URI. For example, the + user is omitted if it is equal to default SDK user given by logging in. The same + is for cluster name: it is omitted if equal to default cluster. + + :param ~yarl.URL uri: an URI to normalize. + + :param ~typing.Iterable[str] allowed_schemes: an *iterable* of accepted URI + schemes, e.g. ``('file', + 'storage')``. No scheme check is + performed by default. + + :param ~typing.Optional[str] cluster_name: optional cluster name, the default + cluster is used if not specified. + + :return: :class:`~yarl.URL` with normalized URI. + + :raise ValueError: if ``uri`` is invalid or provides a scheme not enumerated by + ``allowed_schemes`` argument. + + EnvParseResult ============== diff --git a/neuro-sdk/src/neuro_sdk/parser.py b/neuro-sdk/src/neuro_sdk/parser.py index 0451fc7e1..06fe1ca16 100644 --- a/neuro-sdk/src/neuro_sdk/parser.py +++ b/neuro-sdk/src/neuro_sdk/parser.py @@ -1,14 +1,25 @@ import os import warnings from dataclasses import dataclass -from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, overload +from pathlib import Path +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Optional, + Sequence, + Tuple, + overload, +) from typing_extensions import Literal from yarl import URL from .config import Config from .parsing_utils import LocalImage, RemoteImage, TagOption, _ImageNameParser -from .url_utils import uri_from_cli +from .url_utils import _check_scheme, _extract_path, _normalize_uri, uri_from_cli from .utils import NoPublicConstructor @@ -271,6 +282,57 @@ def volumes( disk_volumes=self._build_disk_volumes(disk_volumes, cluster_name), ) + def uri_to_str(self, uri: URL) -> str: + return str(uri) + + def str_to_uri( + self, + uri: str, + *, + allowed_schemes: Iterable[str] = (), + cluster_name: Optional[str] = None, + ) -> URL: + return uri_from_cli( + uri, + self._config.username, + cluster_name or self._config.cluster_name, + allowed_schemes=allowed_schemes, + ) + + def uri_to_path(self, uri: URL) -> Path: + if uri.scheme != "file": + raise ValueError( + f"Invalid scheme '{uri.scheme}:' (only 'file:' is allowed)" + ) + return _extract_path(uri) + + def path_to_uri( + self, + path: Path, + *, + cluster_name: Optional[str] = None, + ) -> URL: + return uri_from_cli( + str(path), + self._config.username, + cluster_name or self._config.cluster_name, + allowed_schemes=("file",), + ) + + def normalize_uri( + self, + uri: URL, + *, + allowed_schemes: Iterable[str] = (), + cluster_name: Optional[str] = None, + ) -> URL: + _check_scheme(uri.scheme, allowed_schemes) + return _normalize_uri( + uri, + self._config.username, + cluster_name or self._config.cluster_name, + ) + def _read_lines(env_file: str) -> Iterator[str]: with open(env_file, encoding="utf-8-sig") as ef: diff --git a/neuro-sdk/src/neuro_sdk/url_utils.py b/neuro-sdk/src/neuro_sdk/url_utils.py index a297312d4..1a611aadf 100644 --- a/neuro-sdk/src/neuro_sdk/url_utils.py +++ b/neuro-sdk/src/neuro_sdk/url_utils.py @@ -2,7 +2,7 @@ import re import sys from pathlib import Path -from typing import Sequence +from typing import Iterable from urllib.parse import quote_from_bytes from yarl import URL @@ -15,8 +15,10 @@ def uri_from_cli( username: str, cluster_name: str, *, - allowed_schemes: Sequence[str] = ("file", "storage"), + allowed_schemes: Iterable[str] = ("file", "storage"), ) -> URL: + if not isinstance(allowed_schemes, tuple): + allowed_schemes = tuple(allowed_schemes) if "file" in allowed_schemes and path_or_uri.startswith("~"): path_or_uri = os.path.expanduser(path_or_uri) if path_or_uri.startswith("~"): @@ -45,11 +47,7 @@ def uri_from_cli( f"URI Scheme not specified. " f"Please specify one of {', '.join(allowed_schemes)}." ) - if uri.scheme not in allowed_schemes: - raise ValueError( - f"Unsupported URI scheme: {uri.scheme}. " - f"Please specify one of {', '.join(allowed_schemes)}." - ) + _check_scheme(uri.scheme, allowed_schemes) # Check string representation to detect also trailing "?" and "#". _check_uri_str(path_or_uri, uri.scheme) if uri.scheme == "file": @@ -61,6 +59,22 @@ def uri_from_cli( return uri +def _check_scheme(scheme: str, allowed: Iterable[str]) -> None: + if not isinstance(allowed, tuple): + allowed = tuple(allowed) + if not allowed: + return + if scheme not in allowed: + allowed_str = ", ".join(f"'{item}:'" for item in allowed) + if len(allowed) > 1: + verb = "are" + else: + verb = "is" + raise ValueError( + f"Invalid scheme '{scheme}:' (only {allowed_str} {verb} allowed)" + ) + + def normalize_storage_path_uri(uri: URL, username: str, cluster_name: str) -> URL: """Normalize storage url.""" if uri.scheme != "storage":