diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76916c9..68a25af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: unittest: strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] # format: 3.7, 3.8, 3.9 + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12"] # format: 3.7, 3.8, 3.9 platform: [ubuntu-latest, macos-latest, windows-latest] fail-fast: false runs-on: ${{ matrix.platform }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eab022..d9c083a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,17 @@ # envolved Changelog +## 1.5.0 +### Removed +* `envolved` no longer supports python 3.7 +### Added +* `FindIterCollectionParser` +* `with_prefix` can now override many of an env-var's parameters +### Fixed +* `CollectionParser`'s `opener` and `closer` arguments now correctly handle matches that would be split by the delimiter +* `CollectionParser`'s `closer` argument now correctly handles overlapping matches +* `CollectionParser`'s `closer` argument is now faster when using non-regex matches +* `CollectionParser.pair_wise_delimited` will now be more memory efficient when using a mapping `value_type` +### Internal +* fixed some documentation typos ## 1.4.0 ### Deprecated * this is the last release to support python 3.7 diff --git a/docs/cookbook.rst b/docs/cookbook.rst index 152bf6f..b405f7a 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -6,7 +6,7 @@ EnvVars are best defined as global variables (so they will be included in the :ref:`description `). Also, to differentiate the environment variables and their eventually retrieved values, we should end the name of the EnvVar variables with the suffix ``_ev``. -.. code-block:: +.. code-block::python board_size_ev : EnvVar[int] = env_var('BOARD_SIZE', type=int, default=8) @@ -37,7 +37,7 @@ Here are some common types and factories to use when creating a :class:`~envvar. * :class:`typing.NamedTuple`: A quick and easy way to create an annotated named tuple. * :class:`typing.TypedDict`: To create type annotated dictionaries. -.. code-block:: +.. code-block::python class Point(typing.NamedTuple): x: int @@ -72,7 +72,7 @@ Inferring Schema Parameter Names Without a Schema We can actually use :func:`~envvar.inferred_env_var` to infer the name of :class:`~envvar.EnvVar` parameters without a schema. This is useful when we want to prototype a schema without having to create a schema class. -.. code-block:: +.. code-block::python from envolved import ... @@ -87,7 +87,7 @@ we want to prototype a schema without having to create a schema class. Note a sticking point here, we have to specify not only the type of the inferred env var, but also the default value. -.. code-block:: +.. code-block::python from envolved import ... @@ -104,7 +104,7 @@ Note a sticking point here, we have to specify not only the type of the inferred We can specify that an inferred env var is required by explicitly stating `default=missing` -.. code-block:: +.. code-block::python from envolved import ..., missing diff --git a/docs/describing.rst b/docs/describing.rst index fe20ebc..da35ba0 100644 --- a/docs/describing.rst +++ b/docs/describing.rst @@ -3,7 +3,7 @@ Describing Environment Variables Another feature of envolved is the ability to describe all EnvVars. -.. code-block:: +.. code-block::python cache_time_ev = env_var('CACHE_TIME', type=int, default=3600, description='Cache time, in seconds') backlog_size_ev = env_var('BACKLOG_SIZE', type=int, default=100, description='Backlog size') @@ -46,7 +46,7 @@ Excluding EnvVars from the description In some cases it is useful to exclude some EnvVars from the description. This can be done with the :func:`exclude_from_description` function. -.. code-block:: +.. code-block::python point_args = dict( x=env_var('_x', type=int, description='x coordinate'), diff --git a/docs/envvar.rst b/docs/envvar.rst index c3f7c87..1f95125 100644 --- a/docs/envvar.rst +++ b/docs/envvar.rst @@ -93,7 +93,7 @@ EnvVars :param validator: A callable that will be added as a validator. :return: The validator, to allow usage of this function as a decorator. - .. code-block:: + .. code-block::python :caption: Using validators to assert that an environment variable is valid. connection_timeout_ev = env_var('CONNECTION_TIMEOUT_SECONDS', type=int) @@ -105,7 +105,7 @@ EnvVars return value # getting the value of the environment variable will now raise an error if the value is not positive - .. code-block:: + .. code-block::python :caption: Using validators to mutate the value of an environment variable. title_ev = env_var('TITLE', type=str) @@ -119,12 +119,14 @@ EnvVars .. warning:: Even if the validator does not mutate the value, it should still return the original value. - .. method:: with_prefix(prefix: str) -> EnvVar[T] + .. method:: with_prefix(prefix: str, *, default = ..., description = ...) -> EnvVar[T] Return a new EnvVar with the parameters but with a given prefix. This method can be used to re-use an env-var - schema to multiple env-vars. + schema to multiple env-vars. Can also override additional parameters of the new EnvVar. :param prefix: The prefix to use. + :param other: If specified, will override the parameters of the new EnvVar. If not specified, the + parameters of the original EnvVar will be used. Different subclasses can allow to override additional parameters. :return: A new EnvVar with the given prefix, of the same type as the envar being used. .. method:: patch(value: T | missing | discard) -> typing.ContextManager @@ -188,7 +190,7 @@ EnvVars :param kwargs: Additional keyword arguments to pass to the :attr:`type` callable. :return: The value of the retrieved environment variable. - .. code-block:: + .. code-block::python :caption: Using SingleEnvVar to fetch a value from an environment variable, with additional keyword arguments. from dataclasses import dataclass @@ -202,6 +204,10 @@ EnvVars users = users_ev.get(reverse=True) # will return a list of usernames sorted in reverse order else: users = users_ev.get() # will return a list of usernames sorted in ascending order + + .. method:: with_prefix(prefix: str, *, default = ..., description = ..., type = ..., case_sensitive = ..., strip_whitespaces = ...) -> SingleEnvVar[T] + + See :meth:`Superclass method ` .. class:: SchemaEnvVar @@ -256,7 +262,7 @@ EnvVars :param kwargs: Additional keyword arguments to pass to the :attr:`type` callable. :return: The value of the environment variable. - .. code-block:: + .. code-block::python :caption: Using SchemaEnvVar to create a class from multiple environment variables, with additional keyword arguments. from dataclasses import dataclass @@ -273,6 +279,10 @@ EnvVars user_ev.get(age=20, height=168) # will return a User object with the name taken from the environment variables, # but with the age and height overridden by the keyword arguments. + + .. method:: with_prefix(prefix: str, *, default = ..., description = ..., type = ..., on_partial = ...) -> SchemaEnvVar[T] + + See :meth:`Superclass method ` .. class:: Factory(callback: collections.abc.Callable[[], T]) diff --git a/docs/introduction.rst b/docs/introduction.rst index 6b571f4..5b66fac 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -2,7 +2,7 @@ Introduction =============== Envolved is a python library that makes reading and parsing environment variables easy. -.. code-block:: +.. code-block::python from envolved import * @@ -36,7 +36,7 @@ Envolved is a python library that makes reading and parsing environment variable Envolved cuts down on boilerplate and allows for more reusable code. -.. code-block:: +.. code-block::python # If we to accept connection info for another API, we don't need to repeat ourselves diff --git a/docs/string_parsing.rst b/docs/string_parsing.rst index 86a1d28..4d4a32b 100644 --- a/docs/string_parsing.rst +++ b/docs/string_parsing.rst @@ -17,23 +17,23 @@ Some built-in callables translate to special predefined parsers. For example, th ineffective on its own as a parser, which is why envolved knows to treat the ``bool`` type as a special parser that translates the string ``"True"`` and ``"False"`` to ``True`` and ``False`` respectively. -.. code-block:: +.. code-block::python enable_cache_ev = env_var("ENABLE_CACHE", type=bool) os.environ["ENABLE_CACHE"] = "False" - assert enable_cache_ev.get() == False + assert enable_cache_ev.get() is False Users can disable the special meaning of some types by wrapping them in a dummy callable. -.. code-block:: +.. code-block::python enable_cache_ev = env_var("ENABLE_CACHE", type=lambda x: bool(x)) os.environ["ENABLE_CACHE"] = "False" - assert enable_cache_ev.get() == True + assert enable_cache_ev.get() is True All the special parsers are: @@ -75,14 +75,15 @@ Utility Parsers :param delimiter: The delimiter string or pattern to split the string on. :param inner_parser: The parser to use to parse the elements of the collection. Note this parser is treated the same an an EnvVar type, so :ref:`string_parsing:Special parsers` apply. - :param output_type: The type to use to aggregate the parsed items to a collection defaults to list. + :param output_type: The type to use to aggregate the parsed items to a collection. Defaults to list. :param opener: If set, specifies a string or pattern that should be at the beginning of the delimited string. - :param closer: If set, specifies a string or pattern that should be at the end of the delimited string. + :param closer: If set, specifies a string or pattern that should be at the end of the delimited string. Note that providing + a pattern will slow down the parsing process. :param strip: Whether or not to strip whitespaces from the beginning and end of each item. - .. code-block:: + .. code-block::python - countries = env_var("COUNTRIES", type=CollectionParser(",", str.to_lower, set)) + countries = env_var("COUNTRIES", type=CollectionParser(",", str.lower, set)) os.environ["COUNTRIES"] = "United States,Canada,Mexico" @@ -115,18 +116,18 @@ Utility Parsers :param strip_keys: Whether or not to strip whitespaces from the beginning and end of each key in every pair. :param strip_values: Whether or not to strip whitespaces from the beginning and end of each value in every pair. - .. code-block:: + .. code-block::python :caption: Using CollectionParser.pair_wise_delimited to parse arbitrary HTTP headers. headers_ev = env_var("HTTP_HEADERS", - type=CollectionParser.pair_wise_delimited(";", ":", str.to_upper, + type=CollectionParser.pair_wise_delimited(";", ":", str.upper, str)) os.environ["HTTP_HEADERS"] = "Foo:bar;baz:qux" assert headers_ev.get() == {"FOO": "bar", "BAZ": "qux"} - .. code-block:: + .. code-block::python :caption: Using CollectionParser.pair_wise_delimited to parse a key-value collection with differing value types. @@ -140,6 +141,36 @@ Utility Parsers assert server_params_ev.get() == {"host": "localhost", "port": 8080, "is_ssl": False} +.. class:: FindIterCollectionParser(element_pattern: typing.Pattern, element_func: collections.abc.Callable[[re.Match], E], \ + output_type: collections.abc.Callable[[collections.abc.Iterator[E]], G] = list, \ + opener: str | typing.Pattern = '', closer: str | typing.Pattern = '') + + A parser to translate a string to a collection of values by splitting the string to continguous elements that match + a regex pattern. This parser is useful for parsing strings that have a repeating, complex structure, or in cases where + a :class:`naive split ` would split the string incorrectly. + + :param element_pattern: A regex pattern to find the elements in the string. + :param element_func: A function that takes a regex match object and returns an element. + :param output_type: The type to use to aggregate the parsed items to a collection. Defaults to list. + :param opener: If set, specifies a string or pattern that should be at the beginning of the string. + :param closer: If set, specifies a string or pattern that should be at the end of the string. Note that providing + a pattern will slow down the parsing process. + + .. code-block::python + :caption: Using FindIterCollectionParser to parse a string of comma-separated groups of numbers. + + def parse_group(match: re.Match) -> set[int]: + return {int(x) for x in match.group(1).split(',')} + + groups_ev = env_var("GROUPS", type=FindIterCollectionParser( + re.compile(r"{([,\d]+)}(,|$)"), + parse_group + )) + + os.environ["GROUPS"] = "{1,2,3},{4,5,6},{7,8,9}" + + assert groups_ev.get() == [{1, 2, 3}, {4, 5, 6}, {7, 8, 9}] + .. class:: MatchParser(cases: collections.abc.Iterable[tuple[typing.Pattern[str] | str, T]] | \ collections.abc.Mapping[str, T] | type[enum.Enum], fallback: T = ...) @@ -151,7 +182,7 @@ Utility Parsers which case the names of the enum members will be used as the matches. :param fallback: The value to return if no case matches. If not specified, an exception will be raised. - .. code-block:: + .. code-block::python class Color(enum.Enum): RED = 1 @@ -183,7 +214,7 @@ Utility Parsers in which case the names of the enum members will be used as the matches. :param fallback: The value to return if no case matches. If not specified, an exception will be raised. - .. code-block:: + .. code-block::python class Color(enum.Enum): RED = 1 diff --git a/docs/testing_utilities.rst b/docs/testing_utilities.rst index 5b12191..70a0994 100644 --- a/docs/testing_utilities.rst +++ b/docs/testing_utilities.rst @@ -5,7 +5,7 @@ Envolved makes testing environment variables easy with the :attr:`~envvar.EnvVar :meth:`~envvar.EnvVar.patch` context method. They allows you to set a predefined EnvVar value and then restore the original value when the test is finished. -.. code-block:: +.. code-block::python :emphasize-lines: 5-6 cache_time_ev = env_var('CACHE_TIME', type=10) @@ -20,7 +20,7 @@ original value when the test is finished. note that `cache_time_ev.patch(10)` just sets attribute `cache_time_ev.monkeypatch` to ``10``, and restores it to its previous value when the context is exited. We might as well have done: -.. code-block:: +.. code-block::python :emphasize-lines: 5-6, 9 cache_time_ev = env_var('CACHE_TIME', type=10) @@ -40,7 +40,7 @@ Unittest In :mod:`unittest` tests, we can use the :any:`unittest.mock.patch.object` method decorate a test method to the values we want to test with. -.. code-block:: +.. code-block::python :emphasize-lines: 4, 6 cache_time_ev = env_var('CACHE_TIME', type=10) @@ -58,7 +58,7 @@ Pytest When using :mod:`pytest` we can use the `monkeypatch fixture `_ fixture to patch our EnvVars. -.. code-block:: +.. code-block::python :emphasize-lines: 2 def test_app_startup(monkeypatch): @@ -74,7 +74,7 @@ Sometimes we may want to apply a monkeypatch over a non-function-scope fixture. because the built-in monkeypatch fixture is only available in function scope. To overcome this, we can create our own monkeypatch fixture. -.. code-block:: +.. code-block::python from pytest import fixture, MonkeyPatch @@ -98,7 +98,7 @@ monkeypatch fixture. An important thing to note is that the ``monkeypatch`` fixture doesn't affect the actual environment, only the specific EnvVar that was patched. -.. code-block:: +.. code-block::python cache_time_ev = env_var('CACHE_TIME', type=int) @@ -116,7 +116,7 @@ In cases where an environment variable is retrieved from different EnvVars, or w have to set the environment directly, by using the :attr:`envvar.SingleEnvVar.key` property to get the actual environment name. In pytest we can use the monkeypatch fixture to do this. -.. code-block:: +.. code-block::python cache_time_ev = env_var('CACHE_TIME', type=int) diff --git a/envolved/_version.py b/envolved/_version.py index 3e8d9f9..5b60188 100644 --- a/envolved/_version.py +++ b/envolved/_version.py @@ -1 +1 @@ -__version__ = "1.4.0" +__version__ = "1.5.0" diff --git a/envolved/envparser.py b/envolved/envparser.py index b357030..e1dd148 100644 --- a/envolved/envparser.py +++ b/envolved/envparser.py @@ -59,13 +59,11 @@ def __init__(self): class AuditingEnvParser(ReloadingEnvParser): - environ_case_insensitive: MutableMapping[str, Set[str]] - def __init__(self): super().__init__() sys.addaudithook(self.audit_hook) - def audit_hook(self, event: str, args: Tuple[Any, ...]): # pragma: no cover + def audit_hook(self, event: str, args: Tuple[Any, ...]): if event == "os.putenv": if not args: return @@ -120,50 +118,12 @@ def get(self, case_sensitive: bool, key: str) -> str: return ret -class NonAuditingEnvParser(ReloadingEnvParser): - def get(self, case_sensitive: bool, key: str) -> str: - if case_sensitive: - return getenv_unsafe(key) - - def out_of_date() -> str: - self.reload() - return get_case_insensitive(retry_allowed=False) - - lowered = key.lower() - - def get_case_insensitive(retry_allowed: bool) -> str: - if retry_allowed and lowered not in self.environ_case_insensitive: - # if a retry is allowed, and no candidates are available, we need to retry - return out_of_date() - candidates = self.environ_case_insensitive[lowered] - if key in candidates: - preferred_key = key - elif retry_allowed and has_env(key): - # key is not a candidate, but it is in the env - return out_of_date() - elif len(candidates) == 1: - (preferred_key,) = candidates - elif retry_allowed: - return out_of_date() - else: - raise CaseInsensitiveAmbiguityError(candidates) - ret = getenv(preferred_key) - if ret is None: - assert retry_allowed - return out_of_date() - return ret - - return get_case_insensitive(retry_allowed=True) - - EnvParser: Type[BaseEnvParser] if name == "nt": # in windows, all env vars are uppercase EnvParser = CaseInsensitiveEnvParser -elif sys.version_info >= (3, 8): # adding audit hooks is only supported in python 3.8+ - EnvParser = AuditingEnvParser else: - EnvParser = NonAuditingEnvParser + EnvParser = AuditingEnvParser env_parser = EnvParser() diff --git a/envolved/envvar.py b/envolved/envvar.py index b95005d..be18495 100644 --- a/envolved/envvar.py +++ b/envolved/envvar.py @@ -65,6 +65,13 @@ class Discard(Enum): discard = Discard.discard + +class Unchanged(Enum): + unchanged = auto() + + +unchanged = Unchanged.unchanged + Description = Union[str, Sequence[str]] @@ -134,7 +141,13 @@ def _get(self, **kwargs: Any) -> T: pass @abstractmethod - def with_prefix(self: Self, prefix: str) -> Self: + def with_prefix( + self: Self, + prefix: str, + *, + default: Union[T, Factory[T], Missing, Discard, Unchanged] = unchanged, + description: Union[Description, None, Unchanged] = unchanged, + ) -> Self: pass @abstractmethod @@ -194,15 +207,35 @@ def _get(self, **kwargs: Any) -> T: raw_value = raw_value.strip() return self.type(raw_value, **kwargs) - def with_prefix(self, prefix: str) -> SingleEnvVar[T]: + def with_prefix( + self, + prefix: str, + *, + default: Union[T, Factory[T], Missing, Discard, Unchanged] = unchanged, + description: Union[Description, None, Unchanged] = unchanged, + type: Union[Type[T], Parser[T], Unchanged] = unchanged, + case_sensitive: Union[bool, Unchanged] = unchanged, + strip_whitespaces: Union[bool, Unchanged] = unchanged, + ) -> SingleEnvVar[T]: + if default is unchanged: + default = self.default + if description is unchanged: + description = self.description + type_ = type + if type_ is unchanged: + type_ = self.type + if case_sensitive is unchanged: + case_sensitive = self.case_sensitive + if strip_whitespaces is unchanged: + strip_whitespaces = self.strip_whitespaces return register_env_var( SingleEnvVar( prefix + self._key, - self.default, - type=self.type, - description=self.description, - case_sensitive=self.case_sensitive, - strip_whitespaces=self.strip_whitespaces, + default, + type=type_, + description=description, + case_sensitive=case_sensitive, + strip_whitespaces=strip_whitespaces, validators=self._validators, ) ) @@ -292,14 +325,31 @@ def _get(self, **kwargs: Any) -> T: raise errs[0] return self._type(*pos_values, **kw_values) - def with_prefix(self, prefix: str) -> SchemaEnvVar[T]: + def with_prefix( + self, + prefix: str, + *, + default: Union[T, Factory[T], Missing, Discard, Unchanged] = unchanged, + description: Union[Unchanged, None, Description] = unchanged, + type: Union[Type[T], Parser[T], Unchanged] = unchanged, + on_partial: Union[T, Missing, AsDefault, Discard, Factory[T], Unchanged] = unchanged, + ) -> SchemaEnvVar[T]: + if default is unchanged: + default = self.default + if description is unchanged: + description = self.description + type_ = type + if type_ is unchanged: + type_ = self.type + if on_partial is unchanged: + on_partial = self.on_partial return register_env_var( SchemaEnvVar( {k: v.with_prefix(prefix) for k, v in self._args.items()}, - self.default, - type=self._type, - description=self.description, - on_partial=self.on_partial, + default, + type=type_, + description=description, + on_partial=on_partial, validators=self._validators, pos_args=tuple(v.with_prefix(prefix) for v in self._pos_args), ) diff --git a/envolved/parsers.py b/envolved/parsers.py index 801f1ba..8f4daf8 100644 --- a/envolved/parsers.py +++ b/envolved/parsers.py @@ -2,7 +2,6 @@ import re from enum import Enum, auto -from functools import lru_cache from itertools import chain from sys import version_info from typing import ( @@ -138,6 +137,68 @@ def _duplicate_avoiding_dict(pairs: Iterator[Tuple[K, V]]) -> Dict[K, V]: return ret +def strip_opener_idx(x: str, opener: Pattern[str]) -> int: + opener_match = opener.match(x) + if not opener_match: + raise ValueError("position 0, expected opener") + return opener_match.end() + + +def strip_closer_idx(x: str, closer: Needle, pos: int) -> int: + if isinstance(closer, str): + if len(closer) + pos > len(x) or not x.endswith(closer): + raise ValueError("expected string to end in closer") + return len(x) - len(closer) + else: + assert isinstance(closer, Pattern) + # now we have a problem, as the standard re module doesn't support reverse matches + closer_matches = closer.finditer(x, pos) + closer_match = None + for closer_match in closer_matches: # noqa: B007 + # we iterate to find the last match + pass + if not closer_match: + raise ValueError("expected string to end in closer") + else: + while closer_match.end() != len(x): + # finditer could have missed an overlapping match, if there is an overlapping match + # it will be found after the start of the last match (but before its end) + closer_match = closer.search(x, closer_match.start() + 1) + # if there is a match, it's an overlapping match, but it doesn't neccessarily end at + # the end of the string + if not closer_match: + raise ValueError("expected string to end in closer") + return closer_match.start() + + +def strip_opener_and_closer(x: str, opener: Pattern[str], closer: Needle) -> str: + start_idx = strip_opener_idx(x, opener) + end_idx = strip_closer_idx(x, closer, start_idx) + + if start_idx != 0 or end_idx != len(x): + return x[start_idx:end_idx] + return x + + +def value_parser_func(value_type: Union[ParserInput[V], Mapping[K, ParserInput[V]]]) -> Callable[[K], Parser[V]]: + if isinstance(value_type, Mapping): + value_parsers = {k: parser(v) for k, v in value_type.items()} + + def get_value_parser(key: K) -> Parser[V]: + try: + return value_parsers[key] + except KeyError: + # in case the mapping has a default value or the like + return parser(value_type[key]) + else: + _value_parser = parser(value_type) + + def get_value_parser(key: K) -> Parser[V]: + return _value_parser + + return get_value_parser + + class CollectionParser(Generic[G, E]): """ A parser that splits a string by a delimiter, and parses each part individually. @@ -149,45 +210,20 @@ def __init__( inner_parser: ParserInput[E], output_type: Callable[[Iterator[E]], G] = list, # type: ignore[assignment] opener: Needle = empty_pattern, - closer: Needle = empty_pattern, + closer: Needle = "", *, strip: bool = True, ): - """ - :param delimiter: The delimiter to split by. - :param inner_parser: The inner parser to apply to each element. - :param output_type: The aggregator function of all the parsed elements. - :param opener: Optional opener that must be present at the start of the string. - :param closer: Optional closer that must be present at the end of the string. - """ self.delimiter_pattern = needle_to_pattern(delimiter) self.inner_parser = parser(inner_parser) self.output_type = output_type self.opener_pattern = needle_to_pattern(opener) - self.closer_pattern = needle_to_pattern(closer) + self.closer = closer self.strip = strip def __call__(self, x: str) -> G: - opener_match = self.opener_pattern.match(x) - if not opener_match: - raise ValueError("position 0, expected opener") - x = x[opener_match.end() :] - raw_elements = self.delimiter_pattern.split(x) - closer_matches = self.closer_pattern.finditer(raw_elements[-1]) - - closer_match = None - for closer_match in closer_matches: # noqa: B007 - pass - if not closer_match: - raise ValueError("expected string to end in closer") - elif closer_match.end() != len(raw_elements[-1]): - raise ValueError( - "expected closer to match end of string, got unexpected suffix: " - + raw_elements[-1][closer_match.end() :] - ) - - raw_elements[-1] = raw_elements[-1][: closer_match.start()] - raw_items = iter(raw_elements) + x = strip_opener_and_closer(x, self.opener_pattern, self.closer) + raw_items = iter(self.delimiter_pattern.split(x)) if self.strip: raw_items = (r.strip() for r in raw_items) elements = (self.inner_parser(r) for r in raw_items) @@ -201,36 +237,14 @@ def pair_wise_delimited( key_type: ParserInput[K], value_type: Union[ParserInput[V], Mapping[K, ParserInput[V]]], output_type: Callable[[Iterator[Tuple[K, V]]], G] = _duplicate_avoiding_dict, # type: ignore[assignment] - *, key_first: bool = True, strip_keys: bool = True, strip_values: bool = True, **kwargs: Any, ) -> Parser[G]: - """ - Create a collectionParser that aggregates to key-value pairs. - :param pair_delimiter: The separator between different key-value pairs. - :param key_value_delimiter: The separator between each key and value. - :param key_type: The parser for key elements. - :param value_type: The parser for value elements. Can also be a mapping, parsing each key under a different - parser. - :param output_type: The tuple aggregator function. Defaults to a duplicate-checking dict. - :param key_first: If set to false, will evaluate the part behind the key-value separator as a value. - :param kwargs: forwarded to `CollectionParser.__init__` - """ key_value_delimiter = needle_to_pattern(key_value_delimiter) key_parser = parser(key_type) - get_value_parser: Callable[[K], Parser] - if isinstance(value_type, Mapping): - - @lru_cache(None) - def get_value_parser(key: K) -> Parser[V]: - return parser(value_type[key]) - else: - _value_parser = parser(value_type) - - def get_value_parser(key: K) -> Parser[V]: - return _value_parser + get_value_parser = value_parser_func(value_type) def combined_parser(s: str) -> Tuple[K, V]: split = key_value_delimiter.split(s, maxsplit=2) @@ -250,6 +264,38 @@ def combined_parser(s: str) -> Tuple[K, V]: return cls(pair_delimiter, combined_parser, output_type, **kwargs) # type: ignore[arg-type] +def find_iter_contingient(x: str, pattern: Pattern[str]) -> Iterator[re.Match[str]]: + start_idx = 0 + while start_idx < len(x): + match = pattern.match(x, start_idx) + if match is None: + raise ValueError(f"could not match pattern {pattern} at position {start_idx}") + start_idx = match.end() + yield match + + +class FindIterCollectionParser(Generic[G, E]): + def __init__( + self, + element_pattern: Pattern[str], + element_func: Callable[[re.Match[str]], E], + output_type: Callable[[Iterator[E]], G] = list, # type: ignore[assignment] + opener: Needle = empty_pattern, + closer: Needle = "", + ): + self.prefix_pattern = element_pattern + self.element_func = element_func + self.output_type = output_type + self.opener_pattern = needle_to_pattern(opener) + self.closer = closer + + def __call__(self, x: str) -> G: + x = strip_opener_and_closer(x, self.opener_pattern, self.closer) + raw_matches = find_iter_contingient(x, self.prefix_pattern) + elements = (self.element_func(r) for r in raw_matches) + return self.output_type(elements) + + class NoFallback(Enum): no_fallback = auto() diff --git a/pyproject.toml b/pyproject.toml index b28b38a..d81abee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "envolved" -version = "1.4.0" +version = "1.5.0" description = "" authors = ["ben avrahami "] license = "MIT" @@ -12,11 +12,8 @@ packages = [ ] [tool.poetry.dependencies] -python = "^3.7" -typing-extensions = [ - {version="<4.8.0", python=">=3.7, <3.8"}, - {version="*", python=">=3.8"}, -] +python = "^3.8" +typing-extensions = "*" [tool.poetry.group.dev.dependencies] pytest = "*" @@ -39,7 +36,7 @@ build-backend = "poetry.masonry.api" [tool.ruff] -target-version = "py37" +target-version = "py38" line-length = 120 output-format = "full" [tool.ruff.lint] @@ -108,3 +105,7 @@ keep-runtime-typing = true "PTH", # use pathlib "PERF", # performance anti-patterns ] + +"type_checking/**" = [ + "INP001", # implicit namespace packages +] \ No newline at end of file diff --git a/scripts/test_type_hinting.sh b/scripts/test_type_hinting.sh new file mode 100644 index 0000000..5021a42 --- /dev/null +++ b/scripts/test_type_hinting.sh @@ -0,0 +1 @@ +python -m mypy --show-error-codes --check-untyped-defs type_checking \ No newline at end of file diff --git a/tests/unittests/test_examples.py b/tests/unittests/test_examples.py new file mode 100644 index 0000000..717cbfa --- /dev/null +++ b/tests/unittests/test_examples.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import enum +import re +from types import SimpleNamespace + +from pytest import MonkeyPatch, fixture + +from envolved import env_var +from envolved.parsers import CollectionParser, FindIterCollectionParser, LookupParser, MatchParser + + +class FakeEnviron: + def __init__(self, monkeypatch: MonkeyPatch) -> None: + self.monkeypatch = monkeypatch + + def __setitem__(self, key: str, value: str) -> None: + self.monkeypatch.setenv(key, value) + + +@fixture() +def os(): + return SimpleNamespace(environ=FakeEnviron(MonkeyPatch())) + + +def test_bool_special_parser(os): + enable_cache_ev = env_var("ENABLE_CACHE", type=bool) + + os.environ["ENABLE_CACHE"] = "False" + + assert enable_cache_ev.get() is False + + +def test_bypass_bool_parser(os): + enable_cache_ev = env_var("ENABLE_CACHE", type=lambda x: bool(x)) + + os.environ["ENABLE_CACHE"] = "False" + + assert enable_cache_ev.get() is True + + +def test_collection_parser(os): + countries = env_var("COUNTRIES", type=CollectionParser(",", str.lower, set)) + + os.environ["COUNTRIES"] = "United States,Canada,Mexico" + + assert countries.get() == {"united states", "canada", "mexico"} + + +def test_collection_parser_pairwise(os): + headers_ev = env_var("HTTP_HEADERS", type=CollectionParser.pair_wise_delimited(";", ":", str.upper, str)) + + os.environ["HTTP_HEADERS"] = "Foo:bar;baz:qux" + + assert headers_ev.get() == {"FOO": "bar", "BAZ": "qux"} + + +def test_collection_parser_pairwise_2(os): + server_params_ev = env_var( + "SERVER_PARAMS", + type=CollectionParser.pair_wise_delimited( + ";", + ":", + str, + { + "host": str, + "port": int, + "is_ssl": bool, + }, + ), + ) + + os.environ["SERVER_PARAMS"] = "host:localhost;port:8080;is_ssl:false" + + assert server_params_ev.get() == {"host": "localhost", "port": 8080, "is_ssl": False} + + +def test_find_iter_collection_parser(os): + def parse_group(match: re.Match) -> set[int]: + return {int(x) for x in match.group(1).split(",")} + + groups_ev = env_var("GROUPS", type=FindIterCollectionParser(re.compile(r"{([,\d]+)}(,|$)"), parse_group)) + + os.environ["GROUPS"] = "{1,2,3},{4,5,6},{7,8,9}" + + assert groups_ev.get() == [{1, 2, 3}, {4, 5, 6}, {7, 8, 9}] + + +def test_match_parser(os): + class Color(enum.Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + color_ev = env_var("COLOR", type=MatchParser(Color)) + + os.environ["COLOR"] = "RED" + + assert color_ev.get() == Color.RED + + +def test_lookup_parser(os): + class Color(enum.Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + color_ev = env_var("COLOR", type=LookupParser(Color)) + + os.environ["COLOR"] = "RED" + + assert color_ev.get() == Color.RED diff --git a/tests/unittests/test_parsers.py b/tests/unittests/test_parsers.py index e8b7047..b18b0bb 100644 --- a/tests/unittests/test_parsers.py +++ b/tests/unittests/test_parsers.py @@ -1,4 +1,6 @@ import re +from collections import defaultdict +from dataclasses import dataclass from enum import Enum from typing import List @@ -6,7 +8,15 @@ from pydantic.v1 import BaseModel as BaseModel1 from pytest import mark, raises -from envolved.parsers import BoolParser, CollectionParser, LookupParser, MatchParser, complex_parser, parser +from envolved.parsers import ( + BoolParser, + CollectionParser, + FindIterCollectionParser, + LookupParser, + MatchParser, + complex_parser, + parser, +) def test_complex(): @@ -81,6 +91,13 @@ def test_mapping_different_val_types(): assert p("a=hello world;b=true;c=3") == {"a": "hello world", "b": True, "c": 3} +def test_mapping_different_val_types_with_missing(): + val_dict = defaultdict(lambda: str) + val_dict.update({"b": bool, "c": int}) + p = CollectionParser.pair_wise_delimited(";", "=", str, val_dict) + assert p("a=hello world;b=true;c=3") == {"a": "hello world", "b": True, "c": 3} + + def test_mapping_vfirst(): p = CollectionParser.pair_wise_delimited(";", "=", int, str, key_first=False) assert p("a=1;b=2;c=3") == {1: "a", 2: "b", 3: "c"} @@ -263,3 +280,26 @@ def test_typeadapter(): t = TypeAdapter(List[int]) p = parser(t) assert p("[1,2,3]") == [1, 2, 3] + + +@mark.parametrize("closer", ["];]", re.compile(r"\];\]")]) +def test_delimited_boundries_collections(closer): + assert CollectionParser(";", str, opener="[;[", closer=closer)("[;[a;b;c];]") == ["a", "b", "c"] + + +def test_finditer_parser(): + p = FindIterCollectionParser(re.compile(r"\d+(?:\s|$)"), lambda m: int(m[0])) + assert p("1 2 3 4") == [1, 2, 3, 4] + + +def test_finditer_parser_complex(): + @dataclass + class Node: + name: str + values: List[int] + + values_parser = CollectionParser(";", int, opener="(", closer=")") + p = FindIterCollectionParser( + re.compile(r"(\w+)(?:\s*)(\(.*?\))?(;|$)"), lambda m: Node(m[1], values_parser(m[2]) if m[2] else []) + ) + assert p("a(1;2;3);b(4;5;6)") == [Node("a", [1, 2, 3]), Node("b", [4, 5, 6])] diff --git a/tests/unittests/test_parsers_utils.py b/tests/unittests/test_parsers_utils.py new file mode 100644 index 0000000..2b91722 --- /dev/null +++ b/tests/unittests/test_parsers_utils.py @@ -0,0 +1,35 @@ +import re + +from pytest import mark, raises + +from envolved.parsers import strip_opener_and_closer + + +@mark.parametrize("closer", ["]", re.compile(r"\]")]) +def test_strip_bounds(closer): + assert strip_opener_and_closer("[abca]", re.compile(r"\["), closer) == "abca" + + +@mark.parametrize("x", ["[aabc]", "[aabcaaaa]", "[bc]"]) +def test_strip_bounds_dyn(x): + assert strip_opener_and_closer(x, re.compile(r"\[a*"), re.compile(r"a*\]")) == "bc" + + +def test_strip_bounds_overlapping_closer(): + assert strip_opener_and_closer("fababa", re.compile(""), re.compile("aba")) == "fab" + + +def test_strip_no_closer(): + with raises(ValueError): + strip_opener_and_closer("ab", re.compile("a"), re.compile("c")) + + +def test_strip_closer_not_at_end(): + with raises(ValueError): + strip_opener_and_closer("abf", re.compile("a"), re.compile("b")) + + +@mark.parametrize("closer", ["a]", re.compile(r"a\]")]) +def test_strip_no_double_strip(closer): + with raises(ValueError): + strip_opener_and_closer("[a]", re.compile(r"\[a"), closer) diff --git a/tests/unittests/test_single_var.py b/tests/unittests/test_single_var.py index 761399b..f0fea5f 100644 --- a/tests/unittests/test_single_var.py +++ b/tests/unittests/test_single_var.py @@ -158,6 +158,24 @@ def test_override_default(monkeypatch): assert a1.get() == 1 +def test_override_default_in_constr(monkeypatch): + parent = env_var("a", type=int) + + a0 = parent.with_prefix("0") + a1 = parent.with_prefix("1", default=1) + monkeypatch.setenv("0a", "0") + assert a0.get() == 0 + assert a1.get() == 1 + + +def test_override_type(monkeypatch): + parent = env_var("a", type=int) + + a1 = parent.with_prefix("1", default=1, type=len) + assert a1.default == 1 + assert a1.type == len + + def test_patch(): a = env_var("a", type=int) with a.patch(-1): diff --git a/type_checking/env_var.py b/type_checking/env_var.py new file mode 100644 index 0000000..776cc3d --- /dev/null +++ b/type_checking/env_var.py @@ -0,0 +1,29 @@ +from collections.abc import AsyncIterator +from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext + +from envolved import EnvVar, env_var, inferred_env_var +from envolved.describe import exclude_from_description + +number_ev = env_var("NUMBER", type=int) +i: int = number_ev.get() + +ignored_number_ev = exclude_from_description(env_var("IGNORED_NUMBER", type=int)) +j: int = ignored_number_ev.get() + + +# test contravariance +@asynccontextmanager +async def cont(a: int, b: str) -> AsyncIterator[int]: + yield a + + +base_ev: EnvVar[AbstractAsyncContextManager[int | None]] = exclude_from_description( + env_var("SEQ_", type=cont, args={"a": inferred_env_var(), "b": inferred_env_var()}) +) +seq_ev = base_ev.with_prefix("SEQ_") +seq_ev.default = nullcontext() + + +async def test_cont() -> int | None: + async with seq_ev.get() as t: + return t