diff --git a/deker/client.py b/deker/client.py index e3f42c8..75ac531 100644 --- a/deker/client.py +++ b/deker/client.py @@ -142,20 +142,20 @@ def _open(self) -> None: "No installed adapters are found: run `pip install deker_local_adapters`" ) - if self.__uri.scheme not in self.__plugins: + if self.__uri.scheme not in self.__plugins: # type: ignore[attr-defined] raise DekerClientError( - f"Invalid uri: {self.__uri.scheme} is not supported; {self.__uri}" + f"Invalid uri: {self.__uri.scheme} is not supported; {self.__uri}" # type: ignore[attr-defined] ) if self.is_closed: self.__is_closed = False try: - factory = self.__plugins[self.__uri.scheme] + factory = self.__plugins[self.__uri.scheme] # type: ignore[attr-defined] except AttributeError: raise DekerClientError( f"Invalid source: installed package does not provide AdaptersFactory " - f"for managing uri scheme {self.__uri.scheme}" + f"for managing uri scheme {self.__uri.scheme}" # type: ignore[attr-defined] ) self.__ctx = CTX( @@ -267,7 +267,10 @@ def meta_version(self) -> str: @property def root_path(self) -> Path: """Get root path to the current storage.""" - return Path(self.__adapter.uri.path) / self.__config.collections_directory + return ( + Path(self.__adapter.uri.path) # type: ignore[attr-defined] + / self.__config.collections_directory + ) @property def is_closed(self) -> bool: diff --git a/deker/uri.py b/deker/uri.py index d137b8e..f9015a1 100644 --- a/deker/uri.py +++ b/deker/uri.py @@ -16,17 +16,20 @@ from __future__ import annotations -from collections import OrderedDict +from collections import OrderedDict, namedtuple from pathlib import Path -from typing import Dict, List, Union -from urllib.parse import ParseResult, parse_qs, quote, urlparse +from typing import Dict, List, Optional, Tuple, Union +from urllib.parse import ParseResult, _NetlocResultMixinStr, parse_qs, quote, urlparse from deker_tools.path import is_path_valid from deker.errors import DekerValidationError -class Uri(ParseResult): +ParseResultWithServers = namedtuple("ParseResultWithServers", ParseResult._fields + ("servers",)) + + +class Uri(ParseResultWithServers, _NetlocResultMixinStr): """Deker client uri wrapper. Overrides parsed result to have query as a dictionary. @@ -40,18 +43,40 @@ class Uri(ParseResult): params={"separator": ";", "divider": None}, query={"separator": "?", "divider": "&"}, fragment={"separator": "#", "divider": None}, + servers=Optional[List[str]], ) query: Dict[str, List[str]] + servers: List[str] @property def raw_url(self) -> str: """Get url from raw uri without query string, arguments and fragments.""" - url = self.scheme + "://" - if self.netloc: - url += self.netloc - url += quote(str(self.path), safe=":/") + url = self.scheme + "://" # type: ignore[attr-defined] + if self.netloc: # type: ignore[attr-defined] + url += self.netloc # type: ignore[attr-defined] + url += quote(str(self.path), safe=":/") # type: ignore[attr-defined] return url + @classmethod + def __get_servers_and_netloc(cls, netloc: str, scheme: str) -> Tuple[str, Optional[List[str]]]: + """Parse list of servers. + + :param netloc: Netloc object + :param scheme: http or https + """ + # If scheme is not http or https, it couldn't work in cluster mode + if "," not in netloc or scheme not in ["http", "https"]: + # So servers will be None + return netloc, None + + # Otherwise parse servers + servers = netloc.split(",") + node_with_possible_auth = servers[0] + if "@" in node_with_possible_auth: + auth, _ = node_with_possible_auth.split("@") + return node_with_possible_auth, [f"{scheme}://{auth}@{host}" for host in servers[1:]] + return node_with_possible_auth, [f"{scheme}://{host}" for host in servers[1:]] + @classmethod def __parse(cls, uri: str) -> Uri: """Parse uri from string. @@ -66,14 +91,15 @@ def __parse(cls, uri: str) -> Uri: path, params = result.path, result.params query = parse_qs(result.query) - + netloc, servers = cls.__get_servers_and_netloc(result.netloc, result.scheme) return Uri( # type: ignore result.scheme, - result.netloc, + netloc, path, # type: ignore[arg-type] params, query, # type: ignore[arg-type] result.fragment, + servers, ) @classmethod @@ -88,8 +114,8 @@ def validate(cls, uri: str) -> None: raise DekerValidationError("Empty uri passed") parsed_uri = cls.__parse(uri) - if parsed_uri.scheme == "file": - pathname = Path(parsed_uri.path) + if parsed_uri.scheme == "file": # type: ignore[attr-defined] + pathname = Path(parsed_uri.path) # type: ignore[attr-defined] try: is_path_valid(pathname) except Exception as e: @@ -99,7 +125,7 @@ def validate(cls, uri: str) -> None: def create(cls, uri: str) -> Uri: """Create parsed and validated uri from string. - :param uri: : scheme://username:password@host:port/path;parameters?query + :param uri: scheme://username:password@host:port/path;parameters?query """ cls.validate(uri) return cls.__parse(uri) @@ -111,14 +137,16 @@ def __truediv__(self, other: Union[str, Path]) -> Uri: """ sep = "/" other = str(other) - path = sep.join((self.path, other.strip())) + path = sep.join((self.path, other.strip())) # type: ignore[attr-defined] + netloc, servers = self.__get_servers_and_netloc(self.netloc, self.scheme) # type: ignore[attr-defined] res = Uri( # type: ignore - self.scheme, - self.netloc, + self.scheme, # type: ignore[attr-defined] + netloc, path, # type: ignore[arg-type] - self.params, + self.params, # type: ignore[attr-defined] self.query, # type: ignore[arg-type] - self.fragment, + self.fragment, # type: ignore[attr-defined] + servers, ) return res diff --git a/poetry.lock b/poetry.lock index 3d2cfa1..6abd900 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1409,36 +1409,43 @@ setuptools = "*" [[package]] name = "numpy" -version = "1.25.2" +version = "1.26.1" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.9" -files = [ - {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, - {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, - {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, - {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, - {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, - {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, - {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, - {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, - {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, - {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, - {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, - {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, +python-versions = "<3.13,>=3.9" +files = [ + {file = "numpy-1.26.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af"}, + {file = "numpy-1.26.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575"}, + {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244"}, + {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67"}, + {file = "numpy-1.26.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2"}, + {file = "numpy-1.26.1-cp310-cp310-win32.whl", hash = "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297"}, + {file = "numpy-1.26.1-cp310-cp310-win_amd64.whl", hash = "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab"}, + {file = "numpy-1.26.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a"}, + {file = "numpy-1.26.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9"}, + {file = "numpy-1.26.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3"}, + {file = "numpy-1.26.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974"}, + {file = "numpy-1.26.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c"}, + {file = "numpy-1.26.1-cp311-cp311-win32.whl", hash = "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b"}, + {file = "numpy-1.26.1-cp311-cp311-win_amd64.whl", hash = "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53"}, + {file = "numpy-1.26.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f"}, + {file = "numpy-1.26.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24"}, + {file = "numpy-1.26.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e"}, + {file = "numpy-1.26.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124"}, + {file = "numpy-1.26.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c"}, + {file = "numpy-1.26.1-cp312-cp312-win32.whl", hash = "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66"}, + {file = "numpy-1.26.1-cp312-cp312-win_amd64.whl", hash = "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7"}, + {file = "numpy-1.26.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e"}, + {file = "numpy-1.26.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617"}, + {file = "numpy-1.26.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e"}, + {file = "numpy-1.26.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908"}, + {file = "numpy-1.26.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5"}, + {file = "numpy-1.26.1-cp39-cp39-win32.whl", hash = "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104"}, + {file = "numpy-1.26.1-cp39-cp39-win_amd64.whl", hash = "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2"}, + {file = "numpy-1.26.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668"}, + {file = "numpy-1.26.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42"}, + {file = "numpy-1.26.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f"}, + {file = "numpy-1.26.1.tar.gz", hash = "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe"}, ] [[package]] @@ -2648,5 +2655,5 @@ xarray = ["xarray"] [metadata] lock-version = "2.0" -python-versions = "^3.9" -content-hash = "c6e89a1f69e312422c8099412c661af19d485b9249877824c4b0dc3057503011" +python-versions = "^3.9,<3.13" +content-hash = "5ae915a29350d858f821f3be87f66268fe07cdce567b6d36b8b116571a326623" diff --git a/pyproject.toml b/pyproject.toml index 86ef1ef..0a7c07c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,8 +56,7 @@ priority = "supplemental" [tool.poetry.dependencies] -python = "^3.9" -numpy = "^1.18" +python = "^3.9,<3.13" attrs = "^23.1.0" typing-extensions = "^4.4.0" tqdm = "^4.64.1" diff --git a/tests/test_cases/test_uri/test_uri.py b/tests/test_cases/test_uri/test_uri.py index 4879a01..6eb27c6 100644 --- a/tests/test_cases/test_uri/test_uri.py +++ b/tests/test_cases/test_uri/test_uri.py @@ -179,5 +179,30 @@ def test_uri_path_correct_concatenation_with_assignment(string): assert uri.raw_url == "/".join((init_uri.raw_url, "some_path", "some_extra")) +@pytest.mark.parametrize( + "kwargs,result", + ( + ({"netloc": "foo", "scheme": "file"}, ("file://foo", None)), + ({"netloc": "foo", "scheme": "ftp"}, ("ftp://foo", None)), + ( + {"netloc": "foo:bar@host1:8000,host2:8001", "scheme": "http"}, + ("foo:bar@host1:8000", ["http://foo:bar@host2:8001"]), + ), + ( + {"netloc": "host1:8000,host2:8001", "scheme": "http"}, + ("host1:8000", ["http://host2:8001"]), + ), + ), +) +def test_uri_get_netloc_and_servers(kwargs, result): + parsed_netloc, parsed_servers = Uri._Uri__get_servers_and_netloc(**kwargs) + netloc, servers = result + + if result[1]: + assert list(parsed_servers) == servers + else: + assert parsed_servers is None + + if __name__ == "__main__": pytest.main()