From 39337f81c8fb692eb682742392ca8b1a2832c9a9 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 18 Aug 2022 09:38:26 +0200 Subject: [PATCH 01/23] new: usr: add Range Partitioning option to Transformation object --- flycs_sdk/transformations.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flycs_sdk/transformations.py b/flycs_sdk/transformations.py index 4ca7843..50ea679 100644 --- a/flycs_sdk/transformations.py +++ b/flycs_sdk/transformations.py @@ -41,6 +41,7 @@ def __init__( persist_backup: Optional[bool] = None, write_disposition: Optional[WriteDisposition] = WriteDisposition.APPEND, time_partitioning: Optional[dict] = None, + range_partitioning: Optional[dict] = None, cluster_fields: Optional[List[str]] = None, table_expiration: Optional[int] = None, partition_expiration: Optional[int] = None, @@ -80,6 +81,8 @@ def __init__( :type write_disposition: WriteDisposition, optional :param time_partitioning: An dictionary with the time partition configuration, defaults to None :type time_partitioning: dict, optional + :param range_partitioning: A dictionary with the range partition configuration , defaults to None + :type range_partitioning: dict, optional :param cluster_fields: An ordered list of the columns you want to cluster on., defaults to None :type cluster_fields: List[str], optional :param table_expiration: Optional number of day before expiration for the table generated by this transformation @@ -121,6 +124,7 @@ def __init__( self.persist_backup = persist_backup self.write_disposition = write_disposition self.time_partitioning = time_partitioning + self.range_partitioning = range_partitioning self.cluster_fields = cluster_fields self.table_expiration = table_expiration self.partition_expiration = partition_expiration @@ -155,6 +159,7 @@ def from_dict(cls, d: dict): d.get("WRITE_DISPOSITION", "WRITE_APPEND") ), time_partitioning=d.get("TIME_PARTITIONING"), + range_partitioning=d.get("RANGE_PARTITIONING"), cluster_fields=d.get("CLUSTER_FIELDS"), table_expiration=d.get("TABLE_EXPIRATION"), partition_expiration=d.get("PARTITION_EXPIRATION"), @@ -192,6 +197,7 @@ def to_dict(self) -> dict: "PERSIST_BACKUP": self.persist_backup, "WRITE_DISPOSITION": self.write_disposition.value, "TIME_PARTITIONING": self.time_partitioning, + "RANGE_PARTITIONING": self.range_partitioning, "CLUSTER_FIELDS": self.cluster_fields, "TABLE_EXPIRATION": self.table_expiration, "PARTITION_EXPIRATION": self.partition_expiration, @@ -223,6 +229,7 @@ def __eq__(self, other): and self.persist_backup == other.persist_backup and self.write_disposition == other.write_disposition and self.time_partitioning == other.time_partitioning + and self.range_partitioning == other.range_partitioning and self.cluster_fields == other.cluster_fields and self.table_expiration == other.table_expiration and self.partition_expiration == other.partition_expiration From bce95f7c5d1cb71e51a5efd36c69990e91b9331f Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 18 Aug 2022 09:59:27 +0200 Subject: [PATCH 02/23] add comment to range_partitioning field --- flycs_sdk/transformations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flycs_sdk/transformations.py b/flycs_sdk/transformations.py index 50ea679..e55dad5 100644 --- a/flycs_sdk/transformations.py +++ b/flycs_sdk/transformations.py @@ -81,7 +81,7 @@ def __init__( :type write_disposition: WriteDisposition, optional :param time_partitioning: An dictionary with the time partition configuration, defaults to None :type time_partitioning: dict, optional - :param range_partitioning: A dictionary with the range partition configuration , defaults to None + :param range_partitioning: A dictionary with the range partition configuration , defaults to None (time_partitioning must be None) :type range_partitioning: dict, optional :param cluster_fields: An ordered list of the columns you want to cluster on., defaults to None :type cluster_fields: List[str], optional From 0c4617f8463892ee1f9b02a9bde1f67e6b244eca Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 18 Aug 2022 15:01:14 +0200 Subject: [PATCH 03/23] fix: dev: Fix test_to_dict to handle range partitioning field --- tests/test_transformation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_transformation.py b/tests/test_transformation.py index b0f7b9c..6cd5d2f 100644 --- a/tests/test_transformation.py +++ b/tests/test_transformation.py @@ -89,6 +89,7 @@ def test_to_dict(self, my_transformation): "PERSIST_BACKUP": True, "WRITE_DISPOSITION": "WRITE_APPEND", "TIME_PARTITIONING": None, + "RANGE_PARTITIONING": None, "CLUSTER_FIELDS": ["field1", "field2"], "PARTITION_EXPIRATION": None, "REQUIRED_PARTITION_FILTER": False, From b32ebc848221a58eb702eb1f3c90ae72bea599de Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 18 Aug 2022 15:06:23 +0200 Subject: [PATCH 04/23] correct semver to be equal to 2.13.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c73d945..e56a0b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ version = "0.12.0" python = ">=3.8,<3.11" requirements-parser = "^0.5.0" "ruamel.yaml" = "^0.17.0" -semver = "^2.13.0" +semver = "2.13.0" [tool.poetry.dev-dependencies] black = "22.3.0" From 307bb7737f7af0972cdda10c75e785094b6b12c2 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Mon, 22 Aug 2022 12:26:04 +0200 Subject: [PATCH 05/23] new: usr: add a test for multiple partitioning error definition --- flycs_sdk/transformations.py | 12 +++++++++++- tests/test_transformation.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/flycs_sdk/transformations.py b/flycs_sdk/transformations.py index e55dad5..6ad7737 100644 --- a/flycs_sdk/transformations.py +++ b/flycs_sdk/transformations.py @@ -23,8 +23,14 @@ class SchemaUpdateOptions(Enum): ALLOW_FIELD_ADDITION = "ALLOW_FIELD_ADDITION" +class MultiPartitioningDefinitionError(Exception): + """Raised when trying to create a Transformation object with time partitioning & range partitioning.""" + + pass + + class Transformation(QueryBaseWithSchema): - """Transformations are the lowest unit inside of a data pipeline. It is a single task implemented as a SQL query.""" + """Transformations are the lowest unit inside a data pipeline. It is a single task implemented as a SQL query.""" kind = "transformation" @@ -123,6 +129,10 @@ def __init__( self.keep_old_columns = keep_old_columns self.persist_backup = persist_backup self.write_disposition = write_disposition + if time_partitioning and range_partitioning: + raise MultiPartitioningDefinitionError( + "Defining time partitioning & range partitioning is not allowed, you must choose one option between them." + ) self.time_partitioning = time_partitioning self.range_partitioning = range_partitioning self.cluster_fields = cluster_fields diff --git a/tests/test_transformation.py b/tests/test_transformation.py index 6cd5d2f..2be1471 100644 --- a/tests/test_transformation.py +++ b/tests/test_transformation.py @@ -7,6 +7,7 @@ SchemaUpdateOptions, Transformation, WriteDisposition, + MultiPartitioningDefinitionError, ) transformation_name = "my_tranformation" @@ -19,6 +20,7 @@ transformation_persist_backup = True transformation_write_disposition = WriteDisposition.APPEND transformation_time_partitioning = None +transformation_range_partitioning = None transformation_cluster_fields = ["field1", "field2"] transformation_schema_update_options = [SchemaUpdateOptions.ALLOW_FIELD_ADDITION] transformation_dependencies = [Dependency("entity1", "staging", "deps")] @@ -45,6 +47,7 @@ def my_transformation(self): persist_backup=transformation_persist_backup, write_disposition=transformation_write_disposition, time_partitioning=transformation_time_partitioning, + range_partitioning=transformation_range_partitioning, cluster_fields=transformation_cluster_fields, schema_update_options=transformation_schema_update_options, dependencies=transformation_dependencies, @@ -64,6 +67,7 @@ def test_init(self, my_transformation): assert my_transformation.destination_table == transformation_destination_table assert my_transformation.write_disposition == transformation_write_disposition assert my_transformation.time_partitioning == transformation_time_partitioning + assert my_transformation.range_partitioning == transformation_range_partitioning assert my_transformation.cluster_fields == transformation_cluster_fields assert ( my_transformation.schema_update_options @@ -123,3 +127,31 @@ def test_to_dict(self, my_transformation): def test_from_dict(self, my_transformation): loaded = Transformation.from_dict(my_transformation.to_dict()) assert loaded == my_transformation + + def test_mul_partitioning_def_error(self): + with pytest.raises(MultiPartitioningDefinitionError): + transfo = Transformation( + name=transformation_name, + query=transformation_query, + version=transformation_version, + static=transformation_static, + has_output=transformation_has_output, + destination_table=transformation_destination_table, + keep_old_columns=transformation_keep_old_columns, + persist_backup=transformation_persist_backup, + write_disposition=transformation_write_disposition, + time_partitioning={"field": "field1", "type": "DAY"}, + range_partitioning={ + "field": "field2", + "start": 0, + "end": 100, + "interval": 10, + }, + cluster_fields=transformation_cluster_fields, + schema_update_options=transformation_schema_update_options, + dependencies=transformation_dependencies, + parsing_dependencies=transformation_parsing_dependencies, + destroy_table=transformation_destroy_table, + schema=transformation_fields_config, + force_cache_refresh=transformation_force_cache_refresh, + ) From 6970e8e61881bd38f70721e4233a2287cf8789d1 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 25 Aug 2022 15:31:18 +0200 Subject: [PATCH 06/23] new: usr: Add dict schedule object to schedule pipeline based on the environment --- flycs_sdk/pipelines.py | 4 +- poetry.lock | 334 +++++++++++++++-------------------------- tests/test_pipeline.py | 67 +++++++++ 3 files changed, 187 insertions(+), 218 deletions(-) diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index bd57f7b..4ce1e9d 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -38,7 +38,7 @@ def __init__( Entity, BaseLayerEntity, ParametrizedEntity, ParametrizedBaseLayerEntity ] ] = None, - schedule: Optional[str] = None, + schedule: Optional[Union[str, dict]] = None, kind: PipelineKind = PipelineKind.VANILLA, start_time: Optional[datetime] = None, trigger: Optional[PipelineTrigger] = None, @@ -158,7 +158,7 @@ def __init__( name: str, version: str, entities: List[Union[ParametrizedEntity, ParametrizedBaseLayerEntity]] = None, - schedule: Optional[str] = None, + schedule: Optional[Union[str, dict]] = None, kind: PipelineKind = PipelineKind.VANILLA, start_time: Optional[datetime] = None, trigger: Optional[PipelineTrigger] = None, diff --git a/poetry.lock b/poetry.lock index 317f67c..4b743a4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,7 +8,7 @@ python-versions = "*" [[package]] name = "astroid" -version = "2.11.5" +version = "2.11.7" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false @@ -21,7 +21,7 @@ wrapt = ">=1.11,<2" [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "dev" optional = false @@ -29,21 +29,21 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.4.0" +version = "22.1.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "babel" -version = "2.10.1" +version = "2.10.3" description = "Internationalization utilities" category = "dev" optional = false @@ -84,19 +84,19 @@ python-versions = ">=3.5" [[package]] name = "certifi" -version = "2021.10.8" +version = "2022.6.15" description = "Python package for providing Mozilla's CA Bundle." category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "2.0.12" +version = "2.1.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "dev" optional = false -python-versions = ">=3.5.0" +python-versions = ">=3.6.0" [package.extras] unicode_backport = ["unicodedata2"] @@ -114,7 +114,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "dev" optional = false @@ -122,7 +122,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.3.3" +version = "6.4.4" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -147,18 +147,18 @@ cli = ["click (==8.0.3)", "pyyaml (==5.4.1)", "toml (==0.10.2)", "clevercsv (==0 [[package]] name = "dill" -version = "0.3.4" +version = "0.3.5.1" description = "serialize all of python" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" [package.extras] graph = ["objgraph (>=1.7.2)"] [[package]] name = "distlib" -version = "0.3.4" +version = "0.3.5" description = "Distribution utilities" category = "dev" optional = false @@ -166,36 +166,36 @@ python-versions = "*" [[package]] name = "docutils" -version = "0.17.1" +version = "0.19" description = "Docutils -- Python Documentation Utilities" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" [[package]] name = "filelock" -version = "3.7.0" +version = "3.8.0" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] -testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] +testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] [[package]] name = "flake8" -version = "4.0.1" +version = "5.0.4" description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.6.1" [package.dependencies] -mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.8.0,<2.9.0" -pyflakes = ">=2.4.0,<2.5.0" +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.9.0,<2.10.0" +pyflakes = ">=2.5.0,<2.6.0" [[package]] name = "idna" @@ -207,7 +207,7 @@ python-versions = ">=3.5" [[package]] name = "imagesize" -version = "1.3.0" +version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" category = "dev" optional = false @@ -215,7 +215,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "4.11.3" +version = "4.12.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -227,7 +227,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -291,11 +291,11 @@ python-versions = ">=3.7" [[package]] name = "mccabe" -version = "0.6.1" +version = "0.7.0" description = "McCabe checker, plugin for flake8" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "mypy-extensions" @@ -369,48 +369,53 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pycodestyle" -version = "2.8.0" +version = "2.9.1" description = "Python style guide checker" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" [[package]] name = "pyflakes" -version = "2.4.0" +version = "2.5.0" description = "passive checker of Python programs" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [[package]] name = "pygments" -version = "2.12.0" +version = "2.13.0" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false python-versions = ">=3.6" +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "pylint" -version = "2.13.9" +version = "2.14.5" description = "python code static checker" category = "dev" optional = false -python-versions = ">=3.6.2" +python-versions = ">=3.7.2" [package.dependencies] -astroid = ">=2.11.5,<=2.12.0-dev0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} +astroid = ">=2.11.6,<=2.12.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = ">=0.2" isort = ">=4.2.5,<6" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +tomlkit = ">=0.10.1" typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] -testutil = ["gitpython (>3)"] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] [[package]] name = "pyparsing" @@ -446,7 +451,7 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2. [[package]] name = "pytz" -version = "2022.1" +version = "2022.2.1" description = "World timezone definitions, modern and historical" category = "dev" optional = false @@ -454,21 +459,21 @@ python-versions = "*" [[package]] name = "requests" -version = "2.27.1" +version = "2.28.1" description = "Python HTTP for Humans." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} -idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requirements-parser" @@ -530,7 +535,7 @@ python-versions = "*" [[package]] name = "sphinx" -version = "4.5.0" +version = "5.1.1" description = "Python documentation generator" category = "dev" optional = false @@ -540,7 +545,7 @@ python-versions = ">=3.6" alabaster = ">=0.7,<0.8" babel = ">=1.3" colorama = {version = ">=0.3.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.14,<0.18" +docutils = ">=0.14,<0.20" imagesize = "*" importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} Jinja2 = ">=2.3" @@ -557,8 +562,8 @@ sphinxcontrib-serializinghtml = ">=1.1.5" [package.extras] docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "isort", "mypy (>=0.931)", "docutils-stubs", "types-typed-ast", "types-requests"] -test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] +lint = ["flake8 (>=3.5.0)", "flake8-comprehensions", "flake8-bugbear", "isort", "mypy (>=0.971)", "sphinx-lint", "docutils-stubs", "types-typed-ast", "types-requests"] +test = ["pytest (>=4.6)", "html5lib", "cython", "typed-ast"] [[package]] name = "sphinxcontrib-applehelp" @@ -647,9 +652,17 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "tomlkit" +version = "0.11.4" +description = "Style preserving TOML library" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + [[package]] name = "tox" -version = "3.25.0" +version = "3.25.1" description = "tox is a generic virtualenv management and test command line tool" category = "dev" optional = false @@ -671,7 +684,7 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pytest (>=4.0.0)", "pytes [[package]] name = "types-setuptools" -version = "57.4.14" +version = "65.1.0" description = "Typing stubs for setuptools" category = "main" optional = false @@ -679,7 +692,7 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "4.2.0" +version = "4.3.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "dev" optional = false @@ -687,34 +700,33 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.9" +version = "1.26.12" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.14.1" +version = "20.16.3" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] -distlib = ">=0.3.1,<1" -filelock = ">=3.2,<4" -platformdirs = ">=2,<3" -six = ">=1.9.0,<2" +distlib = ">=0.3.5,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" [package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [[package]] name = "wrapt" @@ -734,41 +746,32 @@ python-versions = "*" [[package]] name = "zipp" -version = "3.8.0" +version = "3.8.1" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "c65f7b17f724ab7c4c06944dead3e3cea88b16f43fa0ea613e6bf615c92759b1" +content-hash = "77bac9edfbe60cc2e1c203aeb741b9f364d383d16293f84aa26498987f433338" [metadata.files] alabaster = [ {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] -astroid = [ - {file = "astroid-2.11.5-py3-none-any.whl", hash = "sha256:14ffbb4f6aa2cf474a0834014005487f7ecd8924996083ab411e7fa0b508ce0b"}, - {file = "astroid-2.11.5.tar.gz", hash = "sha256:f4e4ec5294c4b07ac38bab9ca5ddd3914d4bf46f9006eb5c0ae755755061044e"}, -] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] -attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] +astroid = [] +atomicwrites = [] +attrs = [] babel = [ - {file = "Babel-2.10.1-py3-none-any.whl", hash = "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2"}, - {file = "Babel-2.10.1.tar.gz", hash = "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13"}, + {file = "Babel-2.10.3-py3-none-any.whl", hash = "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"}, + {file = "Babel-2.10.3.tar.gz", hash = "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51"}, ] black = [ {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, @@ -800,99 +803,39 @@ bump2version = [ {file = "bump2version-1.0.1.tar.gz", hash = "sha256:762cb2bfad61f4ec8e2bdf452c7c267416f8c70dd9ecb1653fd0bbb01fa936e6"}, ] certifi = [ - {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, - {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, - {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] +charset-normalizer = [] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] -coverage = [ - {file = "coverage-6.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df32ee0f4935a101e4b9a5f07b617d884a531ed5666671ff6ac66d2e8e8246d8"}, - {file = "coverage-6.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75b5dbffc334e0beb4f6c503fb95e6d422770fd2d1b40a64898ea26d6c02742d"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:114944e6061b68a801c5da5427b9173a0dd9d32cd5fcc18a13de90352843737d"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab88a01cd180b5640ccc9c47232e31924d5f9967ab7edd7e5c91c68eee47a69"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad8f9068f5972a46d50fe5f32c09d6ee11da69c560fcb1b4c3baea246ca4109b"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4cd696aa712e6cd16898d63cf66139dc70d998f8121ab558f0e1936396dbc579"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c1a9942e282cc9d3ed522cd3e3cab081149b27ea3bda72d6f61f84eaf88c1a63"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c06455121a089252b5943ea682187a4e0a5cf0a3fb980eb8e7ce394b144430a9"}, - {file = "coverage-6.3.3-cp310-cp310-win32.whl", hash = "sha256:cb5311d6ccbd22578c80028c5e292a7ab9adb91bd62c1982087fad75abe2e63d"}, - {file = "coverage-6.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:6d4a6f30f611e657495cc81a07ff7aa8cd949144e7667c5d3e680d73ba7a70e4"}, - {file = "coverage-6.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:79bf405432428e989cad7b8bc60581963238f7645ae8a404f5dce90236cc0293"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:338c417613f15596af9eb7a39353b60abec9d8ce1080aedba5ecee6a5d85f8d3"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db094a6a4ae6329ed322a8973f83630b12715654c197dd392410400a5bfa1a73"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1414e8b124611bf4df8d77215bd32cba6e3425da8ce9c1f1046149615e3a9a31"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:93b16b08f94c92cab88073ffd185070cdcb29f1b98df8b28e6649145b7f2c90d"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fbc86ae8cc129c801e7baaafe3addf3c8d49c9c1597c44bdf2d78139707c3c62"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b5ba058610e8289a07db2a57bce45a1793ec0d3d11db28c047aae2aa1a832572"}, - {file = "coverage-6.3.3-cp37-cp37m-win32.whl", hash = "sha256:8329635c0781927a2c6ae068461e19674c564e05b86736ab8eb29c420ee7dc20"}, - {file = "coverage-6.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:e5af1feee71099ae2e3b086ec04f57f9950e1be9ecf6c420696fea7977b84738"}, - {file = "coverage-6.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e814a4a5a1d95223b08cdb0f4f57029e8eab22ffdbae2f97107aeef28554517e"}, - {file = "coverage-6.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:61f4fbf3633cb0713437291b8848634ea97f89c7e849c2be17a665611e433f53"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3401b0d2ed9f726fadbfa35102e00d1b3547b73772a1de5508ef3bdbcb36afe7"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8586b177b4407f988731eb7f41967415b2197f35e2a6ee1a9b9b561f6323c8e9"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:892e7fe32191960da559a14536768a62e83e87bbb867e1b9c643e7e0fbce2579"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:afb03f981fadb5aed1ac6e3dd34f0488e1a0875623d557b6fad09b97a942b38a"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cbe91bc84be4e5ef0b1480d15c7b18e29c73bdfa33e07d3725da7d18e1b0aff2"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:91502bf27cbd5c83c95cfea291ef387469f2387508645602e1ca0fd8a4ba7548"}, - {file = "coverage-6.3.3-cp38-cp38-win32.whl", hash = "sha256:c488db059848702aff30aa1d90ef87928d4e72e4f00717343800546fdbff0a94"}, - {file = "coverage-6.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6534fcdfb5c503affb6b1130db7b5bfc8a0f77fa34880146f7a5c117987d0"}, - {file = "coverage-6.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc692c9ee18f0dd3214843779ba6b275ee4bb9b9a5745ba64265bce911aefd1a"}, - {file = "coverage-6.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:462105283de203df8de58a68c1bb4ba2a8a164097c2379f664fa81d6baf94b81"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc972d829ad5ef4d4c5fcabd2bbe2add84ce8236f64ba1c0c72185da3a273130"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:06f54765cdbce99901871d50fe9f41d58213f18e98b170a30ca34f47de7dd5e8"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7835f76a081787f0ca62a53504361b3869840a1620049b56d803a8cb3a9eeea3"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6f5fee77ec3384b934797f1873758f796dfb4f167e1296dc00f8b2e023ce6ee9"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:baa8be8aba3dd1e976e68677be68a960a633a6d44c325757aefaa4d66175050f"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d06380e777dd6b35ee936f333d55b53dc4a8271036ff884c909cf6e94be8b6c"}, - {file = "coverage-6.3.3-cp39-cp39-win32.whl", hash = "sha256:f8cabc5fd0091976ab7b020f5708335033e422de25e20ddf9416bdce2b7e07d8"}, - {file = "coverage-6.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c9441d57b0963cf8340268ad62fc83de61f1613034b79c2b1053046af0c5284"}, - {file = "coverage-6.3.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:d522f1dc49127eab0bfbba4e90fa068ecff0899bbf61bf4065c790ddd6c177fe"}, - {file = "coverage-6.3.3.tar.gz", hash = "sha256:2781c43bffbbec2b8867376d4d61916f5e9c4cc168232528562a61d1b4b01879"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] +coverage = [] deepdiff = [ {file = "deepdiff-5.8.1-py3-none-any.whl", hash = "sha256:e9aea49733f34fab9a0897038d8f26f9d94a97db1790f1b814cced89e9e0d2b7"}, {file = "deepdiff-5.8.1.tar.gz", hash = "sha256:8d4eb2c4e6cbc80b811266419cb71dd95a157094a3947ccf937a94d44943c7b8"}, ] dill = [ - {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"}, - {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"}, -] -distlib = [ - {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, - {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, -] -docutils = [ - {file = "docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"}, - {file = "docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125"}, -] -filelock = [ - {file = "filelock-3.7.0-py3-none-any.whl", hash = "sha256:c7b5fdb219b398a5b28c8e4c1893ef5f98ece6a38c6ab2c22e26ec161556fed6"}, - {file = "filelock-3.7.0.tar.gz", hash = "sha256:b795f1b42a61bbf8ec7113c341dad679d772567b936fbd1bf43c9a238e673e20"}, -] -flake8 = [ - {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, - {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, + {file = "dill-0.3.5.1-py2.py3-none-any.whl", hash = "sha256:33501d03270bbe410c72639b350e941882a8b0fd55357580fbc873fba0c59302"}, + {file = "dill-0.3.5.1.tar.gz", hash = "sha256:d75e41f3eff1eee599d738e76ba8f4ad98ea229db8b085318aa2b3333a208c86"}, ] +distlib = [] +docutils = [] +filelock = [] +flake8 = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -imagesize = [ - {file = "imagesize-1.3.0-py2.py3-none-any.whl", hash = "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c"}, - {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, -] +imagesize = [] importlib-metadata = [ - {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, - {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, + {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, + {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -991,10 +934,7 @@ markupsafe = [ {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] -mccabe = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, -] +mccabe = [] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, @@ -1023,22 +963,10 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pycodestyle = [ - {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, - {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, -] -pyflakes = [ - {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, - {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, -] -pygments = [ - {file = "Pygments-2.12.0-py3-none-any.whl", hash = "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"}, - {file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"}, -] -pylint = [ - {file = "pylint-2.13.9-py3-none-any.whl", hash = "sha256:705c620d388035bdd9ff8b44c5bcdd235bfb49d276d488dd2c8ff1736aa42526"}, - {file = "pylint-2.13.9.tar.gz", hash = "sha256:095567c96e19e6f57b5b907e67d265ff535e588fe26b12b5ebe1fc5645b2c731"}, -] +pycodestyle = [] +pyflakes = [] +pygments = [] +pylint = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1047,14 +975,8 @@ pytest = [ {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, ] -pytz = [ - {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, - {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, -] -requests = [ - {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, - {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, -] +pytz = [] +requests = [] requirements-parser = [ {file = "requirements-parser-0.5.0.tar.gz", hash = "sha256:3336f3a3ae23e06d3f0f88595e4052396e3adf91688787f637e5d2ca1a904069"}, {file = "requirements_parser-0.5.0-py3-none-any.whl", hash = "sha256:e7fcdcd04f2049e73a9fb150d8a0f9d51ce4108f5f7cbeac74c484e17b12bcd9"}, @@ -1102,10 +1024,7 @@ snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] -sphinx = [ - {file = "Sphinx-4.5.0-py3-none-any.whl", hash = "sha256:ebf612653238bcc8f4359627a9b7ce44ede6fdd75d9d30f68255c7383d3a6226"}, - {file = "Sphinx-4.5.0.tar.gz", hash = "sha256:7bf8ca9637a4ee15af412d1a1d9689fec70523a68ca9bb9127c2f3eeb344e2e6"}, -] +sphinx = [] sphinxcontrib-applehelp = [ {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"}, @@ -1138,26 +1057,12 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -tox = [ - {file = "tox-3.25.0-py2.py3-none-any.whl", hash = "sha256:0805727eb4d6b049de304977dfc9ce315a1938e6619c3ab9f38682bb04662a5a"}, - {file = "tox-3.25.0.tar.gz", hash = "sha256:37888f3092aa4e9f835fc8cc6dadbaaa0782651c41ef359e3a5743fcb0308160"}, -] -types-setuptools = [ - {file = "types-setuptools-57.4.14.tar.gz", hash = "sha256:df02fe1dd244f58cf4e67cfc3d0a97930a2d61a72dd89f21d81c71017cd83f9a"}, - {file = "types_setuptools-57.4.14-py3-none-any.whl", hash = "sha256:828f7e7e51e157876f47c80518b23ba0c3c36aa8081efd39d5d39f393938aec9"}, -] -typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, -] -urllib3 = [ - {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, - {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, -] -virtualenv = [ - {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, - {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, -] +tomlkit = [] +tox = [] +types-setuptools = [] +typing-extensions = [] +urllib3 = [] +virtualenv = [] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, @@ -1228,7 +1133,4 @@ yapf = [ {file = "yapf-0.32.0-py2.py3-none-any.whl", hash = "sha256:8fea849025584e486fd06d6ba2bed717f396080fd3cc236ba10cb97c4c51cf32"}, {file = "yapf-0.32.0.tar.gz", hash = "sha256:a3f5085d37ef7e3e004c4ba9f9b3e40c54ff1901cd111f05145ae313a7c67d1b"}, ] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] +zipp = [] diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d3b4ff2..c9e8e80 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -23,6 +23,12 @@ pipeline_kind = PipelineKind.VANILLA pipeline_start_time = datetime.fromtimestamp(1606923514, tz=timezone.utc) pipeline_pubsub_topic = "my_topic" +env_schedule = { + "sbx": pipeline_schedule, + "tst": pipeline_schedule, + "acc": pipeline_schedule, + "prd": None, +} class TestPipeline: @@ -57,6 +63,17 @@ def my_pipeline_pubsub(self, my_entity): entities=[], ) + @pytest.fixture + def my_pipeline_env(self, my_entity): + return Pipeline( + name=pipeline_name, + version=pipeline_version, + schedule=env_schedule, + kind=pipeline_kind, + start_time=pipeline_start_time, + entities=[], + ) + def test_init(self, my_pipeline): assert my_pipeline.name == pipeline_name assert my_pipeline.version == pipeline_version @@ -94,6 +111,14 @@ def test_init_pubsub(self, my_pipeline_pubsub): assert my_pipeline_pubsub.entities == [] assert my_pipeline_pubsub.trigger.topic == pipeline_pubsub_topic + def test_init_env(self, my_pipeline_env): + assert my_pipeline_env.name == pipeline_name + assert my_pipeline_env.version == pipeline_version + assert my_pipeline_env.schedule == env_schedule + assert my_pipeline_env.start_time == pipeline_start_time + assert my_pipeline_env.kind == pipeline_kind + assert my_pipeline_env.entities == [] + def test_invalid_start_time(self): with pytest.raises(TypeError): Pipeline( @@ -189,6 +214,38 @@ def test_to_dict_pubsub(self, my_pipeline_pubsub, my_entity): } assert expected == actual + def test_to_dict_env(self, my_pipeline_env, my_entity): + my_pipeline_env.add_entity(my_entity) + actual = my_pipeline_env.to_dict() + expected = { + "name": pipeline_name, + "version": pipeline_version, + "schedule": env_schedule, + "kind": pipeline_kind.value, + "start_time": "2020-12-02T15:38:34+0000", + "trigger": None, + "params": {}, + "entities": [ + { + "name": "entity1", + "version": "1.0.0", + "kind": None, + "stage_config": [ + { + "name": "raw", + "versions": {"table_1": "1.0.0", "table_2": "1.0.0"}, + }, + { + "name": "staging", + "versions": {"table_1": "1.0.0", "table_2": "1.0.0"}, + }, + ], + "location": None, + } + ], + } + assert expected == actual + def test_serialize_deserialize(self, my_pipeline, my_entity): my_pipeline.add_entity(my_entity) p1 = my_pipeline @@ -216,6 +273,16 @@ def test_serialize_deserialize_pubsub(self, my_pipeline_pubsub, my_entity): if isinstance(my_pipeline_pubsub, ParametrizedPipeline): assert loaded.params # ensure the params area loaded + def test_serialize_deserialize_env(self, my_pipeline_env, my_entity): + my_pipeline_env.add_entity(my_entity) + serialized = my_pipeline_env.to_dict() + if not isinstance(serialized, list): + serialized = [serialized] + for d in serialized: + loaded = Pipeline.from_dict(d) + if isinstance(my_pipeline_env, ParametrizedPipeline): + assert loaded.params # ensure the params area loaded + def test_parse_datetime(self): tstr = _format_datetime(pipeline_start_time) parsed = _parse_datetime(tstr) From a609b489d3d43adda28b8fa3f8c29e5dfd14d923 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Wed, 31 Aug 2022 16:24:15 +0200 Subject: [PATCH 07/23] Allow timezone aware DAG --- flycs_sdk/pipelines.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index bd57f7b..97733fb 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -1,7 +1,8 @@ """Module containing pipeline classes.""" +import pytz import itertools -from datetime import datetime, timezone +from datetime import datetime from enum import Enum from typing import Dict, List, Union, Tuple, Optional @@ -86,7 +87,9 @@ def from_dict(cls, d: dict): name=d["name"], version=d["version"], schedule=d.get("schedule"), - start_time=_parse_datetime(d["start_time"]) + start_time=_parse_datetime( + d["start_time"], d.get("schedule_timezone", "UTC") + ) if d.get("start_time") else None, kind=PipelineKind(d["kind"]), @@ -326,8 +329,10 @@ def _is_valid_start_time(start_time: Optional[datetime]) -> bool: if not isinstance(start_time, datetime): raise TypeError("start_time must be a valid datetime object") - if start_time.tzinfo != timezone.utc: - raise ValueError("start_time timezone must be UTC") + if start_time.tzname() not in pytz.all_timezones: + raise ValueError( + "start_time timezone is invalid , please refers to https://en.wikipedia.org/wiki/List_of_tz_database_time_zones to see available time zones." + ) return True @@ -347,12 +352,19 @@ def _format_datetime(t: datetime) -> str: return t.strftime(_time_format) -def _parse_datetime(tstr: str) -> datetime: +def _parse_datetime(tstr: str, timezone: Optional[str] = "UTC") -> datetime: # ensure we always have the UTC timezone information if not tstr.endswith("+0000"): tstr += "+0000" - return datetime.strptime(tstr, _time_format) + dt = datetime.strptime(tstr, _time_format) + new_dt = None + if timezone in pytz.all_timezones: + new_dt = dt.replace(tzinfo=pytz.timezone(timezone)) + else: + # TODO : Raise an error if timezone is Unknown + pass + return new_dt def format_target_pipeline(p: Pipeline) -> str: From fa9b1b53beee29d826956fd82600005e3ca08b93 Mon Sep 17 00:00:00 2001 From: pedrogdvt <112093467+pedrogdvt@users.noreply.github.com> Date: Thu, 1 Sep 2022 12:46:28 +0100 Subject: [PATCH 08/23] new: dev: add keysets used to transformation (#40) * add keysets used to transformations * reformated file --- flycs_sdk/transformations.py | 7 +++++++ tests/test_transformation.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/flycs_sdk/transformations.py b/flycs_sdk/transformations.py index 6ad7737..0ecf0bc 100644 --- a/flycs_sdk/transformations.py +++ b/flycs_sdk/transformations.py @@ -62,6 +62,7 @@ def __init__( tables: Optional[List[dict]] = None, schema: Optional[List[FieldConfig]] = None, force_cache_refresh: Optional[bool] = False, + keysets_used: Optional[List[str]] = None, ): """Class representing a transformation. @@ -114,6 +115,8 @@ def __init__( :type schema: List[FieldConfig], optional :param force_cache_refresh: whether or not we need to use the cache in the pii service :type force_cache_refresh: bool, optional + :param keysets_used: List of keysets used in the transformation + :type keysets_used: List[str] """ super().__init__( name=name, @@ -145,6 +148,7 @@ def __init__( self.destroy_table = destroy_table self.tables = tables self.force_cache_refresh = force_cache_refresh + self.keysets_used = keysets_used @classmethod def from_dict(cls, d: dict): @@ -186,6 +190,7 @@ def from_dict(cls, d: dict): tables=d.get("TABLES"), schema=[FieldConfig.from_dict(x) for x in d.get("SCHEMA") or []], force_cache_refresh=d.get("FORCE_CACHE_REFRESH", False), + keysets_used=d.get("KEYSETS_USED"), ) def to_dict(self) -> dict: @@ -223,6 +228,7 @@ def to_dict(self) -> dict: "KIND": self.kind, "SCHEMA": [config.to_dict() for config in self.schema], "FORCE_CACHE_REFRESH": self.force_cache_refresh, + "KEYSETS_USED": self.keysets_used, } def __eq__(self, other): @@ -254,4 +260,5 @@ def __eq__(self, other): and self.schema == other.schema and self.force_cache_refresh == other.force_cache_refresh and self.schema == other.schema + and self.keysets_used == other.keysets_used ) diff --git a/tests/test_transformation.py b/tests/test_transformation.py index 2be1471..aa1b8ef 100644 --- a/tests/test_transformation.py +++ b/tests/test_transformation.py @@ -31,6 +31,7 @@ FieldConfig(name="field2", decrypt=True, type="DATE", mode="NULLABLE"), ] transformation_force_cache_refresh = True +transformation_keysets_used = "keyset_1" class TestTranformations: @@ -55,6 +56,7 @@ def my_transformation(self): destroy_table=transformation_destroy_table, schema=transformation_fields_config, force_cache_refresh=transformation_force_cache_refresh, + keysets_used=transformation_keysets_used, ) def test_init(self, my_transformation): @@ -79,6 +81,7 @@ def test_init(self, my_transformation): assert ( my_transformation.force_cache_refresh == transformation_force_cache_refresh ) + assert my_transformation.keysets_used == transformation_keysets_used def test_to_dict(self, my_transformation): assert my_transformation.to_dict() == { @@ -122,6 +125,7 @@ def test_to_dict(self, my_transformation): }, ], "FORCE_CACHE_REFRESH": transformation_force_cache_refresh, + "KEYSETS_USED": "keyset_1", } def test_from_dict(self, my_transformation): From 3f4484efe6ff7588c865a29692fb43e1fa23cda5 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 1 Sep 2022 15:23:03 +0200 Subject: [PATCH 09/23] new: usr: Use pendulum.datetime object for Pipeline TimeZone aware & correct tests [FLYCS-868] --- flycs_sdk/pipelines.py | 43 ++--- poetry.lock | 405 +++++++++++++++++++---------------------- pyproject.toml | 2 +- tests/test_pipeline.py | 28 ++- 4 files changed, 222 insertions(+), 256 deletions(-) diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index 97733fb..0827da4 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -1,6 +1,7 @@ """Module containing pipeline classes.""" import pytz +import pendulum import itertools from datetime import datetime from enum import Enum @@ -87,9 +88,7 @@ def from_dict(cls, d: dict): name=d["name"], version=d["version"], schedule=d.get("schedule"), - start_time=_parse_datetime( - d["start_time"], d.get("schedule_timezone", "UTC") - ) + start_time=_parse_datetime(d["start_time"], d.get("timezone", "UTC")) if d.get("start_time") else None, kind=PipelineKind(d["kind"]), @@ -131,9 +130,7 @@ def to_dict(self) -> Dict: "name": self.name, "version": self.version, "schedule": schedule, - "start_time": _format_datetime(self.start_time) - if self.start_time - else None, + "start_time": f"{self.start_time}" if self.start_time else None, "trigger": self.trigger.to_dict() if self.trigger else None, "kind": self.kind.value, "params": self.params, @@ -283,15 +280,13 @@ def to_dict(self) -> List[Dict]: schedule = self.schedule if isinstance(self.schedule, ParametrizedPipeline): schedule = format_target_pipeline(self.schedule) - + print(self.start_time) return [ { "name": _parametrized_name(self.name, p), "version": self.version, "schedule": schedule, - "start_time": _format_datetime(self.start_time) - if self.start_time - else None, + "start_time": f"{self.start_time}" if self.start_time else None, "trigger": self.trigger.to_dict() if self.trigger else None, "kind": self.kind.value, "params": p, @@ -345,7 +340,7 @@ def _is_valid_trigger(trigger: PipelineTrigger) -> bool: # sine we support python3.6 we cannot use datetime fromisoformat and isoformat methods # instead we use this -_time_format = "%Y-%m-%dT%H:%M:%S%z" +_time_format = "YYYY-MM-DDTHH:mm:ss" def _format_datetime(t: datetime) -> str: @@ -353,18 +348,20 @@ def _format_datetime(t: datetime) -> str: def _parse_datetime(tstr: str, timezone: Optional[str] = "UTC") -> datetime: - - # ensure we always have the UTC timezone information - if not tstr.endswith("+0000"): - tstr += "+0000" - dt = datetime.strptime(tstr, _time_format) - new_dt = None - if timezone in pytz.all_timezones: - new_dt = dt.replace(tzinfo=pytz.timezone(timezone)) - else: - # TODO : Raise an error if timezone is Unknown - pass - return new_dt + """Parse a pendulum datetime with its specific timezone from a string date.""" + if "+" in tstr: + tstr = tstr.split("+")[0] + DeprecationWarning( + "Adding +0000 to specify timezone is decrepated. Now use format start_time format like 2021-09-18T00:00:00. To specify the timezone use 'timezone' attribute" + ) + try: + tmz = pendulum.timezone(timezone) + dt = pendulum.from_format(tstr, _time_format, tz=tmz) + return dt + except: + raise Exception( + "Wrong pendulum datetime configuration, please make sur you're using one of the following timezone name : https://en.wikipedia.org/wiki/List_of_tz_database_time_zones" + ) def format_target_pipeline(p: Pipeline) -> str: diff --git a/poetry.lock b/poetry.lock index 317f67c..8ae3939 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,20 +8,20 @@ python-versions = "*" [[package]] name = "astroid" -version = "2.11.5" +version = "2.12.5" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false -python-versions = ">=3.6.2" +python-versions = ">=3.7.2" [package.dependencies] lazy-object-proxy = ">=1.4.0" typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} -wrapt = ">=1.11,<2" +wrapt = {version = ">=1.11,<2", markers = "python_version < \"3.11\""} [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "dev" optional = false @@ -29,21 +29,21 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.4.0" +version = "22.1.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "babel" -version = "2.10.1" +version = "2.10.3" description = "Internationalization utilities" category = "dev" optional = false @@ -84,19 +84,19 @@ python-versions = ">=3.5" [[package]] name = "certifi" -version = "2021.10.8" +version = "2022.6.15" description = "Python package for providing Mozilla's CA Bundle." category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "2.0.12" +version = "2.1.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "dev" optional = false -python-versions = ">=3.5.0" +python-versions = ">=3.6.0" [package.extras] unicode_backport = ["unicodedata2"] @@ -114,7 +114,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "dev" optional = false @@ -122,7 +122,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.3.3" +version = "6.4.4" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -147,18 +147,18 @@ cli = ["click (==8.0.3)", "pyyaml (==5.4.1)", "toml (==0.10.2)", "clevercsv (==0 [[package]] name = "dill" -version = "0.3.4" +version = "0.3.5.1" description = "serialize all of python" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" [package.extras] graph = ["objgraph (>=1.7.2)"] [[package]] name = "distlib" -version = "0.3.4" +version = "0.3.6" description = "Distribution utilities" category = "dev" optional = false @@ -166,36 +166,36 @@ python-versions = "*" [[package]] name = "docutils" -version = "0.17.1" +version = "0.19" description = "Docutils -- Python Documentation Utilities" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" [[package]] name = "filelock" -version = "3.7.0" +version = "3.8.0" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] -testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] +testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] [[package]] name = "flake8" -version = "4.0.1" +version = "5.0.4" description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.6.1" [package.dependencies] -mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.8.0,<2.9.0" -pyflakes = ">=2.4.0,<2.5.0" +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.9.0,<2.10.0" +pyflakes = ">=2.5.0,<2.6.0" [[package]] name = "idna" @@ -207,7 +207,7 @@ python-versions = ">=3.5" [[package]] name = "imagesize" -version = "1.3.0" +version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" category = "dev" optional = false @@ -215,7 +215,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "4.11.3" +version = "4.12.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -227,7 +227,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -291,11 +291,11 @@ python-versions = ">=3.7" [[package]] name = "mccabe" -version = "0.6.1" +version = "0.7.0" description = "McCabe checker, plugin for flake8" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "mypy-extensions" @@ -329,11 +329,23 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pathspec" -version = "0.9.0" +version = "0.10.0" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.7" + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" [[package]] name = "platformdirs" @@ -369,48 +381,53 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pycodestyle" -version = "2.8.0" +version = "2.9.1" description = "Python style guide checker" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" [[package]] name = "pyflakes" -version = "2.4.0" +version = "2.5.0" description = "passive checker of Python programs" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [[package]] name = "pygments" -version = "2.12.0" +version = "2.13.0" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false python-versions = ">=3.6" +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "pylint" -version = "2.13.9" +version = "2.15.0" description = "python code static checker" category = "dev" optional = false -python-versions = ">=3.6.2" +python-versions = ">=3.7.2" [package.dependencies] -astroid = ">=2.11.5,<=2.12.0-dev0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} +astroid = ">=2.12.4,<=2.14.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = ">=0.2" isort = ">=4.2.5,<6" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +tomlkit = ">=0.10.1" typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] -testutil = ["gitpython (>3)"] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] [[package]] name = "pyparsing" @@ -444,31 +461,50 @@ tomli = ">=1.0.0" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + [[package]] name = "pytz" -version = "2022.1" +version = "2022.2.1" description = "World timezone definitions, modern and historical" category = "dev" optional = false python-versions = "*" +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "requests" -version = "2.27.1" +version = "2.28.1" description = "Python HTTP for Humans." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} -idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requirements-parser" @@ -516,7 +552,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" @@ -530,7 +566,7 @@ python-versions = "*" [[package]] name = "sphinx" -version = "4.5.0" +version = "5.1.1" description = "Python documentation generator" category = "dev" optional = false @@ -540,7 +576,7 @@ python-versions = ">=3.6" alabaster = ">=0.7,<0.8" babel = ">=1.3" colorama = {version = ">=0.3.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.14,<0.18" +docutils = ">=0.14,<0.20" imagesize = "*" importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} Jinja2 = ">=2.3" @@ -557,8 +593,8 @@ sphinxcontrib-serializinghtml = ">=1.1.5" [package.extras] docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "isort", "mypy (>=0.931)", "docutils-stubs", "types-typed-ast", "types-requests"] -test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] +lint = ["flake8 (>=3.5.0)", "flake8-comprehensions", "flake8-bugbear", "isort", "mypy (>=0.971)", "sphinx-lint", "docutils-stubs", "types-typed-ast", "types-requests"] +test = ["pytest (>=4.6)", "html5lib", "cython", "typed-ast"] [[package]] name = "sphinxcontrib-applehelp" @@ -647,9 +683,17 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "tomlkit" +version = "0.11.4" +description = "Style preserving TOML library" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + [[package]] name = "tox" -version = "3.25.0" +version = "3.25.1" description = "tox is a generic virtualenv management and test command line tool" category = "dev" optional = false @@ -671,7 +715,7 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pytest (>=4.0.0)", "pytes [[package]] name = "types-setuptools" -version = "57.4.14" +version = "65.3.0" description = "Typing stubs for setuptools" category = "main" optional = false @@ -679,7 +723,7 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "4.2.0" +version = "4.3.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "dev" optional = false @@ -687,34 +731,33 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.9" +version = "1.26.12" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.14.1" +version = "20.16.4" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] -distlib = ">=0.3.1,<1" -filelock = ">=3.2,<4" -platformdirs = ">=2,<3" -six = ">=1.9.0,<2" +distlib = ">=0.3.5,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" [package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [[package]] name = "wrapt" @@ -734,41 +777,32 @@ python-versions = "*" [[package]] name = "zipp" -version = "3.8.0" +version = "3.8.1" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "c65f7b17f724ab7c4c06944dead3e3cea88b16f43fa0ea613e6bf615c92759b1" +content-hash = "cf182a845c9b22b750c649b36725bf45b541802167886a3c773bf1f1ebfa6866" [metadata.files] alabaster = [ {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] -astroid = [ - {file = "astroid-2.11.5-py3-none-any.whl", hash = "sha256:14ffbb4f6aa2cf474a0834014005487f7ecd8924996083ab411e7fa0b508ce0b"}, - {file = "astroid-2.11.5.tar.gz", hash = "sha256:f4e4ec5294c4b07ac38bab9ca5ddd3914d4bf46f9006eb5c0ae755755061044e"}, -] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] -attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] +astroid = [] +atomicwrites = [] +attrs = [] babel = [ - {file = "Babel-2.10.1-py3-none-any.whl", hash = "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2"}, - {file = "Babel-2.10.1.tar.gz", hash = "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13"}, + {file = "Babel-2.10.3-py3-none-any.whl", hash = "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"}, + {file = "Babel-2.10.3.tar.gz", hash = "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51"}, ] black = [ {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, @@ -800,99 +834,39 @@ bump2version = [ {file = "bump2version-1.0.1.tar.gz", hash = "sha256:762cb2bfad61f4ec8e2bdf452c7c267416f8c70dd9ecb1653fd0bbb01fa936e6"}, ] certifi = [ - {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, - {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, - {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] +charset-normalizer = [] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] -coverage = [ - {file = "coverage-6.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df32ee0f4935a101e4b9a5f07b617d884a531ed5666671ff6ac66d2e8e8246d8"}, - {file = "coverage-6.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75b5dbffc334e0beb4f6c503fb95e6d422770fd2d1b40a64898ea26d6c02742d"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:114944e6061b68a801c5da5427b9173a0dd9d32cd5fcc18a13de90352843737d"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab88a01cd180b5640ccc9c47232e31924d5f9967ab7edd7e5c91c68eee47a69"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad8f9068f5972a46d50fe5f32c09d6ee11da69c560fcb1b4c3baea246ca4109b"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4cd696aa712e6cd16898d63cf66139dc70d998f8121ab558f0e1936396dbc579"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c1a9942e282cc9d3ed522cd3e3cab081149b27ea3bda72d6f61f84eaf88c1a63"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c06455121a089252b5943ea682187a4e0a5cf0a3fb980eb8e7ce394b144430a9"}, - {file = "coverage-6.3.3-cp310-cp310-win32.whl", hash = "sha256:cb5311d6ccbd22578c80028c5e292a7ab9adb91bd62c1982087fad75abe2e63d"}, - {file = "coverage-6.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:6d4a6f30f611e657495cc81a07ff7aa8cd949144e7667c5d3e680d73ba7a70e4"}, - {file = "coverage-6.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:79bf405432428e989cad7b8bc60581963238f7645ae8a404f5dce90236cc0293"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:338c417613f15596af9eb7a39353b60abec9d8ce1080aedba5ecee6a5d85f8d3"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db094a6a4ae6329ed322a8973f83630b12715654c197dd392410400a5bfa1a73"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1414e8b124611bf4df8d77215bd32cba6e3425da8ce9c1f1046149615e3a9a31"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:93b16b08f94c92cab88073ffd185070cdcb29f1b98df8b28e6649145b7f2c90d"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fbc86ae8cc129c801e7baaafe3addf3c8d49c9c1597c44bdf2d78139707c3c62"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b5ba058610e8289a07db2a57bce45a1793ec0d3d11db28c047aae2aa1a832572"}, - {file = "coverage-6.3.3-cp37-cp37m-win32.whl", hash = "sha256:8329635c0781927a2c6ae068461e19674c564e05b86736ab8eb29c420ee7dc20"}, - {file = "coverage-6.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:e5af1feee71099ae2e3b086ec04f57f9950e1be9ecf6c420696fea7977b84738"}, - {file = "coverage-6.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e814a4a5a1d95223b08cdb0f4f57029e8eab22ffdbae2f97107aeef28554517e"}, - {file = "coverage-6.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:61f4fbf3633cb0713437291b8848634ea97f89c7e849c2be17a665611e433f53"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3401b0d2ed9f726fadbfa35102e00d1b3547b73772a1de5508ef3bdbcb36afe7"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8586b177b4407f988731eb7f41967415b2197f35e2a6ee1a9b9b561f6323c8e9"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:892e7fe32191960da559a14536768a62e83e87bbb867e1b9c643e7e0fbce2579"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:afb03f981fadb5aed1ac6e3dd34f0488e1a0875623d557b6fad09b97a942b38a"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cbe91bc84be4e5ef0b1480d15c7b18e29c73bdfa33e07d3725da7d18e1b0aff2"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:91502bf27cbd5c83c95cfea291ef387469f2387508645602e1ca0fd8a4ba7548"}, - {file = "coverage-6.3.3-cp38-cp38-win32.whl", hash = "sha256:c488db059848702aff30aa1d90ef87928d4e72e4f00717343800546fdbff0a94"}, - {file = "coverage-6.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6534fcdfb5c503affb6b1130db7b5bfc8a0f77fa34880146f7a5c117987d0"}, - {file = "coverage-6.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc692c9ee18f0dd3214843779ba6b275ee4bb9b9a5745ba64265bce911aefd1a"}, - {file = "coverage-6.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:462105283de203df8de58a68c1bb4ba2a8a164097c2379f664fa81d6baf94b81"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc972d829ad5ef4d4c5fcabd2bbe2add84ce8236f64ba1c0c72185da3a273130"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:06f54765cdbce99901871d50fe9f41d58213f18e98b170a30ca34f47de7dd5e8"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7835f76a081787f0ca62a53504361b3869840a1620049b56d803a8cb3a9eeea3"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6f5fee77ec3384b934797f1873758f796dfb4f167e1296dc00f8b2e023ce6ee9"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:baa8be8aba3dd1e976e68677be68a960a633a6d44c325757aefaa4d66175050f"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d06380e777dd6b35ee936f333d55b53dc4a8271036ff884c909cf6e94be8b6c"}, - {file = "coverage-6.3.3-cp39-cp39-win32.whl", hash = "sha256:f8cabc5fd0091976ab7b020f5708335033e422de25e20ddf9416bdce2b7e07d8"}, - {file = "coverage-6.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c9441d57b0963cf8340268ad62fc83de61f1613034b79c2b1053046af0c5284"}, - {file = "coverage-6.3.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:d522f1dc49127eab0bfbba4e90fa068ecff0899bbf61bf4065c790ddd6c177fe"}, - {file = "coverage-6.3.3.tar.gz", hash = "sha256:2781c43bffbbec2b8867376d4d61916f5e9c4cc168232528562a61d1b4b01879"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] +coverage = [] deepdiff = [ {file = "deepdiff-5.8.1-py3-none-any.whl", hash = "sha256:e9aea49733f34fab9a0897038d8f26f9d94a97db1790f1b814cced89e9e0d2b7"}, {file = "deepdiff-5.8.1.tar.gz", hash = "sha256:8d4eb2c4e6cbc80b811266419cb71dd95a157094a3947ccf937a94d44943c7b8"}, ] dill = [ - {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"}, - {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"}, -] -distlib = [ - {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, - {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, -] -docutils = [ - {file = "docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"}, - {file = "docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125"}, -] -filelock = [ - {file = "filelock-3.7.0-py3-none-any.whl", hash = "sha256:c7b5fdb219b398a5b28c8e4c1893ef5f98ece6a38c6ab2c22e26ec161556fed6"}, - {file = "filelock-3.7.0.tar.gz", hash = "sha256:b795f1b42a61bbf8ec7113c341dad679d772567b936fbd1bf43c9a238e673e20"}, -] -flake8 = [ - {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, - {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, + {file = "dill-0.3.5.1-py2.py3-none-any.whl", hash = "sha256:33501d03270bbe410c72639b350e941882a8b0fd55357580fbc873fba0c59302"}, + {file = "dill-0.3.5.1.tar.gz", hash = "sha256:d75e41f3eff1eee599d738e76ba8f4ad98ea229db8b085318aa2b3333a208c86"}, ] +distlib = [] +docutils = [] +filelock = [] +flake8 = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -imagesize = [ - {file = "imagesize-1.3.0-py2.py3-none-any.whl", hash = "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c"}, - {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, -] +imagesize = [] importlib-metadata = [ - {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, - {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, + {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, + {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -991,10 +965,7 @@ markupsafe = [ {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] -mccabe = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, -] +mccabe = [] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, @@ -1007,9 +978,29 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pathspec = [ - {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, - {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, +pathspec = [] +pendulum = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, ] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, @@ -1023,22 +1014,10 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pycodestyle = [ - {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, - {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, -] -pyflakes = [ - {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, - {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, -] -pygments = [ - {file = "Pygments-2.12.0-py3-none-any.whl", hash = "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"}, - {file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"}, -] -pylint = [ - {file = "pylint-2.13.9-py3-none-any.whl", hash = "sha256:705c620d388035bdd9ff8b44c5bcdd235bfb49d276d488dd2c8ff1736aa42526"}, - {file = "pylint-2.13.9.tar.gz", hash = "sha256:095567c96e19e6f57b5b907e67d265ff535e588fe26b12b5ebe1fc5645b2c731"}, -] +pycodestyle = [] +pyflakes = [] +pygments = [] +pylint = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1047,14 +1026,16 @@ pytest = [ {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, ] -pytz = [ - {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, - {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] -requests = [ - {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, - {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, +pytz = [] +pytzdata = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, ] +requests = [] requirements-parser = [ {file = "requirements-parser-0.5.0.tar.gz", hash = "sha256:3336f3a3ae23e06d3f0f88595e4052396e3adf91688787f637e5d2ca1a904069"}, {file = "requirements_parser-0.5.0-py3-none-any.whl", hash = "sha256:e7fcdcd04f2049e73a9fb150d8a0f9d51ce4108f5f7cbeac74c484e17b12bcd9"}, @@ -1102,10 +1083,7 @@ snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] -sphinx = [ - {file = "Sphinx-4.5.0-py3-none-any.whl", hash = "sha256:ebf612653238bcc8f4359627a9b7ce44ede6fdd75d9d30f68255c7383d3a6226"}, - {file = "Sphinx-4.5.0.tar.gz", hash = "sha256:7bf8ca9637a4ee15af412d1a1d9689fec70523a68ca9bb9127c2f3eeb344e2e6"}, -] +sphinx = [] sphinxcontrib-applehelp = [ {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"}, @@ -1138,26 +1116,12 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -tox = [ - {file = "tox-3.25.0-py2.py3-none-any.whl", hash = "sha256:0805727eb4d6b049de304977dfc9ce315a1938e6619c3ab9f38682bb04662a5a"}, - {file = "tox-3.25.0.tar.gz", hash = "sha256:37888f3092aa4e9f835fc8cc6dadbaaa0782651c41ef359e3a5743fcb0308160"}, -] -types-setuptools = [ - {file = "types-setuptools-57.4.14.tar.gz", hash = "sha256:df02fe1dd244f58cf4e67cfc3d0a97930a2d61a72dd89f21d81c71017cd83f9a"}, - {file = "types_setuptools-57.4.14-py3-none-any.whl", hash = "sha256:828f7e7e51e157876f47c80518b23ba0c3c36aa8081efd39d5d39f393938aec9"}, -] -typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, -] -urllib3 = [ - {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, - {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, -] -virtualenv = [ - {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, - {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, -] +tomlkit = [] +tox = [] +types-setuptools = [] +typing-extensions = [] +urllib3 = [] +virtualenv = [] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, @@ -1228,7 +1192,4 @@ yapf = [ {file = "yapf-0.32.0-py2.py3-none-any.whl", hash = "sha256:8fea849025584e486fd06d6ba2bed717f396080fd3cc236ba10cb97c4c51cf32"}, {file = "yapf-0.32.0.tar.gz", hash = "sha256:a3f5085d37ef7e3e004c4ba9f9b3e40c54ff1901cd111f05145ae313a7c67d1b"}, ] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] +zipp = [] diff --git a/pyproject.toml b/pyproject.toml index e56a0b5..f5e0ed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ python = ">=3.8,<3.11" requirements-parser = "^0.5.0" "ruamel.yaml" = "^0.17.0" semver = "2.13.0" - +pendulum="^2.1.1" [tool.poetry.dev-dependencies] black = "22.3.0" bump2version = "*" diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d3b4ff2..30c724d 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,6 +5,8 @@ from datetime import datetime, timezone, timedelta import pytest +import pytz +import pendulum from deepdiff import DeepDiff from flycs_sdk.entities import Entity, ParametrizedEntity from flycs_sdk.pipelines import ( @@ -21,7 +23,7 @@ pipeline_version = "1.0.0" pipeline_schedule = "* 12 * * *" pipeline_kind = PipelineKind.VANILLA -pipeline_start_time = datetime.fromtimestamp(1606923514, tz=timezone.utc) +pipeline_start_time = pendulum.from_format("2020-12-02T15:38:34", "YYYY-MM-DDTHH:mm:ss") pipeline_pubsub_topic = "my_topic" @@ -129,7 +131,7 @@ def test_to_dict(self, my_pipeline, my_entity): "version": pipeline_version, "schedule": pipeline_schedule, "kind": pipeline_kind.value, - "start_time": "2020-12-02T15:38:34+0000", + "start_time": "2020-12-02T15:38:34+00:00", "trigger": None, "params": {}, "entities": [ @@ -161,7 +163,7 @@ def test_to_dict_pubsub(self, my_pipeline_pubsub, my_entity): "version": pipeline_version, "schedule": pipeline_schedule, "kind": pipeline_kind.value, - "start_time": "2020-12-02T15:38:34+0000", + "start_time": "2020-12-02T15:38:34+00:00", "trigger": { "type": "pubsub", "topic": "my_topic", @@ -212,14 +214,20 @@ def test_serialize_deserialize_pubsub(self, my_pipeline_pubsub, my_entity): if not isinstance(serialized, list): serialized = [serialized] for d in serialized: + print(d) loaded = Pipeline.from_dict(d) if isinstance(my_pipeline_pubsub, ParametrizedPipeline): assert loaded.params # ensure the params area loaded def test_parse_datetime(self): - tstr = _format_datetime(pipeline_start_time) - parsed = _parse_datetime(tstr) - assert parsed == pipeline_start_time + tstr = "2021-09-18T00:00:00" + correct_dt = pendulum.datetime( + 2021, 9, 18, 0, 0, 0, tz=pendulum.timezone("Europe/Brussels") + ) + parsed = _parse_datetime(tstr, "Europe/Brussels") + print(f"{parsed}") + assert parsed.tzinfo == correct_dt.tzinfo + assert parsed == correct_dt pipeline_parameters = {"language": ["nl", "fr"], "country": ["be", "en"]} @@ -271,7 +279,7 @@ def test_to_dict(self, my_pipeline, my_entity): "name": "test_nl_be", "version": "1.0.0", "schedule": "* 12 * * *", - "start_time": "2020-12-02T15:38:34+0000", + "start_time": "2020-12-02T15:38:34+00:00", "trigger": None, "kind": "vanilla", "params": {"language": "nl", "country": "be"}, @@ -298,7 +306,7 @@ def test_to_dict(self, my_pipeline, my_entity): "name": "test_nl_en", "version": "1.0.0", "schedule": "* 12 * * *", - "start_time": "2020-12-02T15:38:34+0000", + "start_time": "2020-12-02T15:38:34+00:00", "trigger": None, "kind": "vanilla", "params": {"language": "nl", "country": "en"}, @@ -325,7 +333,7 @@ def test_to_dict(self, my_pipeline, my_entity): "name": "test_fr_be", "version": "1.0.0", "schedule": "* 12 * * *", - "start_time": "2020-12-02T15:38:34+0000", + "start_time": "2020-12-02T15:38:34+00:00", "trigger": None, "kind": "vanilla", "params": {"language": "fr", "country": "be"}, @@ -352,7 +360,7 @@ def test_to_dict(self, my_pipeline, my_entity): "name": "test_fr_en", "version": "1.0.0", "schedule": "* 12 * * *", - "start_time": "2020-12-02T15:38:34+0000", + "start_time": "2020-12-02T15:38:34+00:00", "trigger": None, "kind": "vanilla", "params": {"language": "fr", "country": "en"}, From 9c49ada49df1856e58ac50fbb4f2f5f1cc1a59f3 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Mon, 5 Sep 2022 13:05:36 +0200 Subject: [PATCH 10/23] remove print(start_time) --- flycs_sdk/pipelines.py | 1 - tests/test_pipeline.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index 0827da4..b31050d 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -280,7 +280,6 @@ def to_dict(self) -> List[Dict]: schedule = self.schedule if isinstance(self.schedule, ParametrizedPipeline): schedule = format_target_pipeline(self.schedule) - print(self.start_time) return [ { "name": _parametrized_name(self.name, p), diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 30c724d..7b73da7 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,9 +5,7 @@ from datetime import datetime, timezone, timedelta import pytest -import pytz import pendulum -from deepdiff import DeepDiff from flycs_sdk.entities import Entity, ParametrizedEntity from flycs_sdk.pipelines import ( Pipeline, From a862ae36e02fbbc7efc885c54effdd4a35b21d3e Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Mon, 5 Sep 2022 14:15:52 +0200 Subject: [PATCH 11/23] use start_time.timezone_name instead of tzname() in validation of start time --- flycs_sdk/pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index b31050d..f8f3f62 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -323,7 +323,7 @@ def _is_valid_start_time(start_time: Optional[datetime]) -> bool: if not isinstance(start_time, datetime): raise TypeError("start_time must be a valid datetime object") - if start_time.tzname() not in pytz.all_timezones: + if start_time.timezone_name not in pytz.all_timezones: raise ValueError( "start_time timezone is invalid , please refers to https://en.wikipedia.org/wiki/List_of_tz_database_time_zones to see available time zones." ) From addaf54b2dc150cb868b639f6bb0631615bea439 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Tue, 6 Sep 2022 09:08:18 +0200 Subject: [PATCH 12/23] new: usr: Add run_before keyset & func_kwargs arg in CustomCode obj --- flycs_sdk/custom_code.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/flycs_sdk/custom_code.py b/flycs_sdk/custom_code.py index 689b5d0..8302aca 100644 --- a/flycs_sdk/custom_code.py +++ b/flycs_sdk/custom_code.py @@ -1,7 +1,7 @@ """Module containing class used to inject custom Airflow operator into pipelines definitions.""" import inspect -from typing import Callable, List +from typing import Callable, Dict, List from requirements.requirement import Requirement @@ -75,6 +75,8 @@ def __init__( operator_builder: Callable, dependencies: List[Dependency] = None, requirements: List[str] = None, + func_kwargs: Dict[str, object] = None, + run_before_keyset: bool = False, ): """Represent a custom Airflow code that needs to be injected into a DAG. @@ -90,19 +92,26 @@ def __init__( :param requirements: list of python package required by this code, use the same format as normal python requirements.txt files. These package will be installed on the composer instance. :type requirements: List[str] + :param func_kwargs: List of kwargs areguments for a customer operator + :type func_kwargs: Dict[str:object] + :param run_before_keyset: override dependencies and only use the explicitly defined dependencies + :type run_before_keyset: bool """ self.name = name self.version = version self.operator_builder = operator_builder self.dependencies = dependencies or [] self.requirements = requirements or [] + self.func_kwargs = func_kwargs or {} + self.run_before_keyset = run_before_keyset or False self._ensure_builder_signature(operator_builder) self._validate_requirements() def _ensure_builder_signature(self, f: Callable): signature = inspect.Signature.from_callable(f) - if ["dag", "env"] != list(signature.parameters.keys()): + signature_parameters = list(signature.parameters.keys()) + if "dag" not in signature_parameters and "env" not in signature_parameters: raise WrongSignatureError( f"the builder function of the custom code {self.name}_{self.version} does not accept the mandatory ('dag', 'env', 'user') arguments" ) From d648f19e41c9f7c6ac6fee1a312945aa9ad60614 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Tue, 6 Sep 2022 09:34:57 +0200 Subject: [PATCH 13/23] include run_before_keyset into Transformation obj --- flycs_sdk/transformations.py | 7 +++++++ tests/test_transformation.py | 1 + 2 files changed, 8 insertions(+) diff --git a/flycs_sdk/transformations.py b/flycs_sdk/transformations.py index 0ecf0bc..6e54046 100644 --- a/flycs_sdk/transformations.py +++ b/flycs_sdk/transformations.py @@ -60,6 +60,7 @@ def __init__( parsing_dependencies: Optional[List[Dependency]] = None, destroy_table: Optional[bool] = False, tables: Optional[List[dict]] = None, + run_before_keyset: Optional[bool] = False, schema: Optional[List[FieldConfig]] = None, force_cache_refresh: Optional[bool] = False, keysets_used: Optional[List[str]] = None, @@ -111,6 +112,8 @@ def __init__( :param tables: If specified, this transformation will generate multiple BigQueryOperator during Airflow generation, one for each table name in this list. The name of the transformation then becomes `{transformation_name}_{table_name}` :type tables: List[str], optional + :param run_before_keyset: override dependencies and only use the explicitly defined dependencies + :type run_before_keyset: bool :param schema: List of extra configuration per field of the transformation :type schema: List[FieldConfig], optional :param force_cache_refresh: whether or not we need to use the cache in the pii service @@ -149,6 +152,7 @@ def __init__( self.tables = tables self.force_cache_refresh = force_cache_refresh self.keysets_used = keysets_used + self.run_before_keyset = run_before_keyset @classmethod def from_dict(cls, d: dict): @@ -188,6 +192,7 @@ def from_dict(cls, d: dict): ], destroy_table=d.get("DESTROY_TABLE", False), tables=d.get("TABLES"), + run_before_keyset=d.get("RUN_BEFORE_KEYSET"), schema=[FieldConfig.from_dict(x) for x in d.get("SCHEMA") or []], force_cache_refresh=d.get("FORCE_CACHE_REFRESH", False), keysets_used=d.get("KEYSETS_USED"), @@ -226,6 +231,7 @@ def to_dict(self) -> dict: "DESTROY_TABLE": self.destroy_table, "TABLES": self.tables, "KIND": self.kind, + "RUN_BEFORE_KEYSET": self.run_before_keyset, "SCHEMA": [config.to_dict() for config in self.schema], "FORCE_CACHE_REFRESH": self.force_cache_refresh, "KEYSETS_USED": self.keysets_used, @@ -257,6 +263,7 @@ def __eq__(self, other): and self.destroy_table == other.destroy_table and self.tables == other.tables and self.kind == other.kind + and self.run_before_keyset == other.run_before_keyset and self.schema == other.schema and self.force_cache_refresh == other.force_cache_refresh and self.schema == other.schema diff --git a/tests/test_transformation.py b/tests/test_transformation.py index aa1b8ef..64b359a 100644 --- a/tests/test_transformation.py +++ b/tests/test_transformation.py @@ -124,6 +124,7 @@ def test_to_dict(self, my_transformation): "FIELDS": [], }, ], + "RUN_BEFORE_KEYSET": False, "FORCE_CACHE_REFRESH": transformation_force_cache_refresh, "KEYSETS_USED": "keyset_1", } From e1af7143af640c7bb498bd749dbb4381d8d120eb Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Tue, 6 Sep 2022 10:11:54 +0200 Subject: [PATCH 14/23] include pytz package into dependencies --- poetry.lock | 25 ++++++------------------- pyproject.toml | 1 + 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8ae3939..90a3bb7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,7 +8,7 @@ python-versions = "*" [[package]] name = "astroid" -version = "2.12.5" +version = "2.12.6" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false @@ -19,14 +19,6 @@ lazy-object-proxy = ">=1.4.0" typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} wrapt = {version = ">=1.11,<2", markers = "python_version < \"3.11\""} -[[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "attrs" version = "22.1.0" @@ -329,7 +321,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pathspec" -version = "0.10.0" +version = "0.10.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false @@ -442,14 +434,13 @@ diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pytest" -version = "7.1.2" +version = "7.1.3" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" @@ -476,7 +467,7 @@ six = ">=1.5" name = "pytz" version = "2022.2.1" description = "World timezone definitions, modern and historical" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -790,7 +781,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "cf182a845c9b22b750c649b36725bf45b541802167886a3c773bf1f1ebfa6866" +content-hash = "7ad1d288740638adb5ab0be90d278a3266b9746ca9a216cde9a7ee83ba8236a5" [metadata.files] alabaster = [ @@ -798,7 +789,6 @@ alabaster = [ {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] astroid = [] -atomicwrites = [] attrs = [] babel = [ {file = "Babel-2.10.3-py3-none-any.whl", hash = "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"}, @@ -1022,10 +1012,7 @@ pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] -pytest = [ - {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, - {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, -] +pytest = [] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, diff --git a/pyproject.toml b/pyproject.toml index f5e0ed8..9050a30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ requirements-parser = "^0.5.0" "ruamel.yaml" = "^0.17.0" semver = "2.13.0" pendulum="^2.1.1" +pytz="2022.2.1" [tool.poetry.dev-dependencies] black = "22.3.0" bump2version = "*" From af5cf2c3fa521dfff106c94cf9ad6637a9dc26f6 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 8 Sep 2022 11:47:13 +0200 Subject: [PATCH 15/23] Support both datetime & pendulum datetime object for python pipeline --- flycs_sdk/pipelines.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index f8f3f62..7afadd5 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -42,7 +42,7 @@ def __init__( ] = None, schedule: Optional[str] = None, kind: PipelineKind = PipelineKind.VANILLA, - start_time: Optional[datetime] = None, + start_time: Optional[Union[datetime, pendulum.DateTime]] = None, trigger: Optional[PipelineTrigger] = None, params: Optional[Dict[str, str]] = None, ): @@ -160,7 +160,7 @@ def __init__( entities: List[Union[ParametrizedEntity, ParametrizedBaseLayerEntity]] = None, schedule: Optional[str] = None, kind: PipelineKind = PipelineKind.VANILLA, - start_time: Optional[datetime] = None, + start_time: Optional[Union[datetime, pendulum.DateTime]] = None, trigger: Optional[PipelineTrigger] = None, parameters: Dict[str, List[str]] = None, ): @@ -308,7 +308,9 @@ def _is_valid_version(version: str) -> bool: return True -def _is_valid_start_time(start_time: Optional[datetime]) -> bool: +def _is_valid_start_time( + start_time: Optional[Union[datetime, pendulum.DateTime]] +) -> bool: """Test if start_time is a valid timestamp value. :param start_time: timestamp to validate @@ -320,10 +322,17 @@ def _is_valid_start_time(start_time: Optional[datetime]) -> bool: if start_time is None: return True - if not isinstance(start_time, datetime): - raise TypeError("start_time must be a valid datetime object") + if not isinstance(start_time, datetime) and not isinstance( + start_time, pendulum.DateTime + ): + raise TypeError( + "start_time must be a valid datetime or pendulum.datetime object" + ) - if start_time.timezone_name not in pytz.all_timezones: + if ( + isinstance(start_time, pendulum.DateTime) + and start_time.timezone_name not in pytz.all_timezones + ): raise ValueError( "start_time timezone is invalid , please refers to https://en.wikipedia.org/wiki/List_of_tz_database_time_zones to see available time zones." ) @@ -346,7 +355,7 @@ def _format_datetime(t: datetime) -> str: return t.strftime(_time_format) -def _parse_datetime(tstr: str, timezone: Optional[str] = "UTC") -> datetime: +def _parse_datetime(tstr: str, timezone: Optional[str] = "UTC") -> pendulum.DateTime: """Parse a pendulum datetime with its specific timezone from a string date.""" if "+" in tstr: tstr = tstr.split("+")[0] From 705a539d788d73857c3b48b507ea9e28c24f13e7 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Thu, 8 Sep 2022 15:08:45 +0200 Subject: [PATCH 16/23] print DeprecationWarning +0000 timezone config in start_time yaml attribute --- flycs_sdk/pipelines.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index 7afadd5..431fba2 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -359,8 +359,10 @@ def _parse_datetime(tstr: str, timezone: Optional[str] = "UTC") -> pendulum.Date """Parse a pendulum datetime with its specific timezone from a string date.""" if "+" in tstr: tstr = tstr.split("+")[0] - DeprecationWarning( - "Adding +0000 to specify timezone is decrepated. Now use format start_time format like 2021-09-18T00:00:00. To specify the timezone use 'timezone' attribute" + print( + DeprecationWarning( + "Adding +0000 to specify timezone is decrepated. Now use format start_time format like 2021-09-18T00:00:00. To specify the timezone use 'timezone' attribute" + ) ) try: tmz = pendulum.timezone(timezone) From b46adc2115fa4f7797b2a97b8b06150a92a53fe4 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis <95035433+yannis-arg@users.noreply.github.com> Date: Fri, 9 Sep 2022 11:14:02 +0200 Subject: [PATCH 17/23] Update flycs_sdk/custom_code.py Co-authored-by: christophedcpm <75361109+christophedcpm@users.noreply.github.com> --- flycs_sdk/custom_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flycs_sdk/custom_code.py b/flycs_sdk/custom_code.py index 8302aca..40a8b45 100644 --- a/flycs_sdk/custom_code.py +++ b/flycs_sdk/custom_code.py @@ -92,7 +92,7 @@ def __init__( :param requirements: list of python package required by this code, use the same format as normal python requirements.txt files. These package will be installed on the composer instance. :type requirements: List[str] - :param func_kwargs: List of kwargs areguments for a customer operator + :param func_kwargs: List of key words arguments for a custom operator :type func_kwargs: Dict[str:object] :param run_before_keyset: override dependencies and only use the explicitly defined dependencies :type run_before_keyset: bool From 6694e3bfdddca98ad884f02e22d9980203f94470 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Fri, 9 Sep 2022 11:41:53 +0200 Subject: [PATCH 18/23] Resolve MR review --- flycs_sdk/custom_code.py | 4 ++-- flycs_sdk/pipelines.py | 13 ++++++++++--- flycs_sdk/transformations.py | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/flycs_sdk/custom_code.py b/flycs_sdk/custom_code.py index 8302aca..af2502a 100644 --- a/flycs_sdk/custom_code.py +++ b/flycs_sdk/custom_code.py @@ -92,9 +92,9 @@ def __init__( :param requirements: list of python package required by this code, use the same format as normal python requirements.txt files. These package will be installed on the composer instance. :type requirements: List[str] - :param func_kwargs: List of kwargs areguments for a customer operator + :param func_kwargs: List of kwargs arguments for a customer operator. E.g {"key_int" : 10} :type func_kwargs: Dict[str:object] - :param run_before_keyset: override dependencies and only use the explicitly defined dependencies + :param run_before_keyset: overrides dependencies and runs the custom Operator before the keysets (keysets are Operators run before the Transformations when PII is activated) :type run_before_keyset: bool """ self.name = name diff --git a/flycs_sdk/pipelines.py b/flycs_sdk/pipelines.py index 431fba2..c8bf32b 100644 --- a/flycs_sdk/pipelines.py +++ b/flycs_sdk/pipelines.py @@ -131,6 +131,9 @@ def to_dict(self) -> Dict: "version": self.version, "schedule": schedule, "start_time": f"{self.start_time}" if self.start_time else None, + "timezone": self.start_time.timezone_name + if isinstance(self.start_time, pendulum.DateTime) + else None, "trigger": self.trigger.to_dict() if self.trigger else None, "kind": self.kind.value, "params": self.params, @@ -286,6 +289,9 @@ def to_dict(self) -> List[Dict]: "version": self.version, "schedule": schedule, "start_time": f"{self.start_time}" if self.start_time else None, + "timezone": self.start_time.timezone_name + if isinstance(self.start_time, pendulum.DateTime) + else None, "trigger": self.trigger.to_dict() if self.trigger else None, "kind": self.kind.value, "params": p, @@ -334,7 +340,7 @@ def _is_valid_start_time( and start_time.timezone_name not in pytz.all_timezones ): raise ValueError( - "start_time timezone is invalid , please refers to https://en.wikipedia.org/wiki/List_of_tz_database_time_zones to see available time zones." + f"timezone attribute {start_time.timezone_name} is invalid , please refers to https://en.wikipedia.org/wiki/List_of_tz_database_time_zones to see available time zones." ) return True @@ -369,8 +375,9 @@ def _parse_datetime(tstr: str, timezone: Optional[str] = "UTC") -> pendulum.Date dt = pendulum.from_format(tstr, _time_format, tz=tmz) return dt except: - raise Exception( - "Wrong pendulum datetime configuration, please make sur you're using one of the following timezone name : https://en.wikipedia.org/wiki/List_of_tz_database_time_zones" + raise ValueError( + "Wrong datetime configuration, please make sur you're using one of the following timezone name : https://en.wikipedia.org/wiki/List_of_tz_database_time_zones" + "E.g. start_time : '2022-09-06T10:00:00' ; timezone : 'Europe/Brussels' " ) diff --git a/flycs_sdk/transformations.py b/flycs_sdk/transformations.py index 6e54046..b19ea53 100644 --- a/flycs_sdk/transformations.py +++ b/flycs_sdk/transformations.py @@ -112,7 +112,7 @@ def __init__( :param tables: If specified, this transformation will generate multiple BigQueryOperator during Airflow generation, one for each table name in this list. The name of the transformation then becomes `{transformation_name}_{table_name}` :type tables: List[str], optional - :param run_before_keyset: override dependencies and only use the explicitly defined dependencies + :param run_before_keyset: overrides dependencies and runs the custom Operator before the keysets (keysets are Operators run before the Transformations when PII is activated) :type run_before_keyset: bool :param schema: List of extra configuration per field of the transformation :type schema: List[FieldConfig], optional From eaf56681519e7773c21d7f57f96c059cadf37176 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Fri, 9 Sep 2022 12:44:43 +0200 Subject: [PATCH 19/23] correct to_dict deserialization test --- tests/test_pipeline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 7b73da7..8be3f55 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -130,6 +130,7 @@ def test_to_dict(self, my_pipeline, my_entity): "schedule": pipeline_schedule, "kind": pipeline_kind.value, "start_time": "2020-12-02T15:38:34+00:00", + "timezone": "UTC", "trigger": None, "params": {}, "entities": [ @@ -162,6 +163,7 @@ def test_to_dict_pubsub(self, my_pipeline_pubsub, my_entity): "schedule": pipeline_schedule, "kind": pipeline_kind.value, "start_time": "2020-12-02T15:38:34+00:00", + "timezone": "UTC", "trigger": { "type": "pubsub", "topic": "my_topic", @@ -278,6 +280,7 @@ def test_to_dict(self, my_pipeline, my_entity): "version": "1.0.0", "schedule": "* 12 * * *", "start_time": "2020-12-02T15:38:34+00:00", + "timezone": "UTC", "trigger": None, "kind": "vanilla", "params": {"language": "nl", "country": "be"}, @@ -305,6 +308,7 @@ def test_to_dict(self, my_pipeline, my_entity): "version": "1.0.0", "schedule": "* 12 * * *", "start_time": "2020-12-02T15:38:34+00:00", + "timezone": "UTC", "trigger": None, "kind": "vanilla", "params": {"language": "nl", "country": "en"}, @@ -332,6 +336,7 @@ def test_to_dict(self, my_pipeline, my_entity): "version": "1.0.0", "schedule": "* 12 * * *", "start_time": "2020-12-02T15:38:34+00:00", + "timezone": "UTC", "trigger": None, "kind": "vanilla", "params": {"language": "fr", "country": "be"}, @@ -359,6 +364,7 @@ def test_to_dict(self, my_pipeline, my_entity): "version": "1.0.0", "schedule": "* 12 * * *", "start_time": "2020-12-02T15:38:34+00:00", + "timezone": "UTC", "trigger": None, "kind": "vanilla", "params": {"language": "fr", "country": "en"}, From fab7809192cd0e09dccb60edfc4b98576c840601 Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Date: Fri, 9 Sep 2022 14:41:00 +0200 Subject: [PATCH 20/23] update poetry.lock --- poetry.lock | 420 +++++++++++++++++++++++++++++----------------------- 1 file changed, 236 insertions(+), 184 deletions(-) diff --git a/poetry.lock b/poetry.lock index 90a3bb7..317f67c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,34 +8,42 @@ python-versions = "*" [[package]] name = "astroid" -version = "2.12.6" +version = "2.11.5" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false -python-versions = ">=3.7.2" +python-versions = ">=3.6.2" [package.dependencies] lazy-object-proxy = ">=1.4.0" typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} -wrapt = {version = ">=1.11,<2", markers = "python_version < \"3.11\""} +wrapt = ">=1.11,<2" + +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "22.1.0" +version = "21.4.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "babel" -version = "2.10.3" +version = "2.10.1" description = "Internationalization utilities" category = "dev" optional = false @@ -76,19 +84,19 @@ python-versions = ">=3.5" [[package]] name = "certifi" -version = "2022.6.15" +version = "2021.10.8" description = "Python package for providing Mozilla's CA Bundle." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = "*" [[package]] name = "charset-normalizer" -version = "2.1.1" +version = "2.0.12" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "dev" optional = false -python-versions = ">=3.6.0" +python-versions = ">=3.5.0" [package.extras] unicode_backport = ["unicodedata2"] @@ -106,7 +114,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.4" description = "Cross-platform colored terminal text." category = "dev" optional = false @@ -114,7 +122,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.4.4" +version = "6.3.3" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -139,18 +147,18 @@ cli = ["click (==8.0.3)", "pyyaml (==5.4.1)", "toml (==0.10.2)", "clevercsv (==0 [[package]] name = "dill" -version = "0.3.5.1" +version = "0.3.4" description = "serialize all of python" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +python-versions = ">=2.7, !=3.0.*" [package.extras] graph = ["objgraph (>=1.7.2)"] [[package]] name = "distlib" -version = "0.3.6" +version = "0.3.4" description = "Distribution utilities" category = "dev" optional = false @@ -158,36 +166,36 @@ python-versions = "*" [[package]] name = "docutils" -version = "0.19" +version = "0.17.1" description = "Docutils -- Python Documentation Utilities" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "filelock" -version = "3.8.0" +version = "3.7.0" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] -testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] +testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] [[package]] name = "flake8" -version = "5.0.4" +version = "4.0.1" description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false -python-versions = ">=3.6.1" +python-versions = ">=3.6" [package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.9.0,<2.10.0" -pyflakes = ">=2.5.0,<2.6.0" +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.8.0,<2.9.0" +pyflakes = ">=2.4.0,<2.5.0" [[package]] name = "idna" @@ -199,7 +207,7 @@ python-versions = ">=3.5" [[package]] name = "imagesize" -version = "1.4.1" +version = "1.3.0" description = "Getting image size from png/jpeg/jpeg2000/gif file" category = "dev" optional = false @@ -207,7 +215,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "4.12.0" +version = "4.11.3" description = "Read metadata from Python packages" category = "dev" optional = false @@ -219,7 +227,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -283,11 +291,11 @@ python-versions = ">=3.7" [[package]] name = "mccabe" -version = "0.7.0" +version = "0.6.1" description = "McCabe checker, plugin for flake8" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = "*" [[package]] name = "mypy-extensions" @@ -321,23 +329,11 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pathspec" -version = "0.10.1" +version = "0.9.0" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false -python-versions = ">=3.7" - -[[package]] -name = "pendulum" -version = "2.1.2" -description = "Python datetimes made easy" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.dependencies] -python-dateutil = ">=2.6,<3.0" -pytzdata = ">=2020.1" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [[package]] name = "platformdirs" @@ -373,53 +369,48 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pycodestyle" -version = "2.9.1" +version = "2.8.0" description = "Python style guide checker" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pyflakes" -version = "2.5.0" +version = "2.4.0" description = "passive checker of Python programs" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.13.0" +version = "2.12.0" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false python-versions = ">=3.6" -[package.extras] -plugins = ["importlib-metadata"] - [[package]] name = "pylint" -version = "2.15.0" +version = "2.13.9" description = "python code static checker" category = "dev" optional = false -python-versions = ">=3.7.2" +python-versions = ">=3.6.2" [package.dependencies] -astroid = ">=2.12.4,<=2.14.0-dev0" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +astroid = ">=2.11.5,<=2.12.0-dev0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} dill = ">=0.2" isort = ">=4.2.5,<6" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -tomlkit = ">=0.10.1" typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] -spelling = ["pyenchant (>=3.2,<4.0)"] -testutils = ["gitpython (>3)"] +testutil = ["gitpython (>3)"] [[package]] name = "pyparsing" @@ -434,13 +425,14 @@ diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pytest" -version = "7.1.3" +version = "7.1.2" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" @@ -452,50 +444,31 @@ tomli = ">=1.0.0" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - [[package]] name = "pytz" -version = "2022.2.1" +version = "2022.1" description = "World timezone definitions, modern and historical" -category = "main" +category = "dev" optional = false python-versions = "*" -[[package]] -name = "pytzdata" -version = "2020.1" -description = "The Olson timezone database for Python." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "requests" -version = "2.28.1" +version = "2.27.1" description = "Python HTTP for Humans." category = "dev" optional = false -python-versions = ">=3.7, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" -idna = ">=2.5,<4" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "requirements-parser" @@ -543,7 +516,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" @@ -557,7 +530,7 @@ python-versions = "*" [[package]] name = "sphinx" -version = "5.1.1" +version = "4.5.0" description = "Python documentation generator" category = "dev" optional = false @@ -567,7 +540,7 @@ python-versions = ">=3.6" alabaster = ">=0.7,<0.8" babel = ">=1.3" colorama = {version = ">=0.3.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.14,<0.20" +docutils = ">=0.14,<0.18" imagesize = "*" importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} Jinja2 = ">=2.3" @@ -584,8 +557,8 @@ sphinxcontrib-serializinghtml = ">=1.1.5" [package.extras] docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "flake8-comprehensions", "flake8-bugbear", "isort", "mypy (>=0.971)", "sphinx-lint", "docutils-stubs", "types-typed-ast", "types-requests"] -test = ["pytest (>=4.6)", "html5lib", "cython", "typed-ast"] +lint = ["flake8 (>=3.5.0)", "isort", "mypy (>=0.931)", "docutils-stubs", "types-typed-ast", "types-requests"] +test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] [[package]] name = "sphinxcontrib-applehelp" @@ -674,17 +647,9 @@ category = "dev" optional = false python-versions = ">=3.7" -[[package]] -name = "tomlkit" -version = "0.11.4" -description = "Style preserving TOML library" -category = "dev" -optional = false -python-versions = ">=3.6,<4.0" - [[package]] name = "tox" -version = "3.25.1" +version = "3.25.0" description = "tox is a generic virtualenv management and test command line tool" category = "dev" optional = false @@ -706,7 +671,7 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pytest (>=4.0.0)", "pytes [[package]] name = "types-setuptools" -version = "65.3.0" +version = "57.4.14" description = "Typing stubs for setuptools" category = "main" optional = false @@ -714,7 +679,7 @@ python-versions = "*" [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.2.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "dev" optional = false @@ -722,33 +687,34 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.12" +version = "1.26.9" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.16.4" +version = "20.14.1" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] -distlib = ">=0.3.5,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<3" +distlib = ">=0.3.1,<1" +filelock = ">=3.2,<4" +platformdirs = ">=2,<3" +six = ">=1.9.0,<2" [package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] +docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] [[package]] name = "wrapt" @@ -768,31 +734,41 @@ python-versions = "*" [[package]] name = "zipp" -version = "3.8.1" +version = "3.8.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "7ad1d288740638adb5ab0be90d278a3266b9746ca9a216cde9a7ee83ba8236a5" +content-hash = "c65f7b17f724ab7c4c06944dead3e3cea88b16f43fa0ea613e6bf615c92759b1" [metadata.files] alabaster = [ {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] -astroid = [] -attrs = [] +astroid = [ + {file = "astroid-2.11.5-py3-none-any.whl", hash = "sha256:14ffbb4f6aa2cf474a0834014005487f7ecd8924996083ab411e7fa0b508ce0b"}, + {file = "astroid-2.11.5.tar.gz", hash = "sha256:f4e4ec5294c4b07ac38bab9ca5ddd3914d4bf46f9006eb5c0ae755755061044e"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] babel = [ - {file = "Babel-2.10.3-py3-none-any.whl", hash = "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb"}, - {file = "Babel-2.10.3.tar.gz", hash = "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51"}, + {file = "Babel-2.10.1-py3-none-any.whl", hash = "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2"}, + {file = "Babel-2.10.1.tar.gz", hash = "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13"}, ] black = [ {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, @@ -824,39 +800,99 @@ bump2version = [ {file = "bump2version-1.0.1.tar.gz", hash = "sha256:762cb2bfad61f4ec8e2bdf452c7c267416f8c70dd9ecb1653fd0bbb01fa936e6"}, ] certifi = [ - {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, - {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, + {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, + {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, + {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, ] -charset-normalizer = [] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +coverage = [ + {file = "coverage-6.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df32ee0f4935a101e4b9a5f07b617d884a531ed5666671ff6ac66d2e8e8246d8"}, + {file = "coverage-6.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75b5dbffc334e0beb4f6c503fb95e6d422770fd2d1b40a64898ea26d6c02742d"}, + {file = "coverage-6.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:114944e6061b68a801c5da5427b9173a0dd9d32cd5fcc18a13de90352843737d"}, + {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab88a01cd180b5640ccc9c47232e31924d5f9967ab7edd7e5c91c68eee47a69"}, + {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad8f9068f5972a46d50fe5f32c09d6ee11da69c560fcb1b4c3baea246ca4109b"}, + {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4cd696aa712e6cd16898d63cf66139dc70d998f8121ab558f0e1936396dbc579"}, + {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c1a9942e282cc9d3ed522cd3e3cab081149b27ea3bda72d6f61f84eaf88c1a63"}, + {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c06455121a089252b5943ea682187a4e0a5cf0a3fb980eb8e7ce394b144430a9"}, + {file = "coverage-6.3.3-cp310-cp310-win32.whl", hash = "sha256:cb5311d6ccbd22578c80028c5e292a7ab9adb91bd62c1982087fad75abe2e63d"}, + {file = "coverage-6.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:6d4a6f30f611e657495cc81a07ff7aa8cd949144e7667c5d3e680d73ba7a70e4"}, + {file = "coverage-6.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:79bf405432428e989cad7b8bc60581963238f7645ae8a404f5dce90236cc0293"}, + {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:338c417613f15596af9eb7a39353b60abec9d8ce1080aedba5ecee6a5d85f8d3"}, + {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db094a6a4ae6329ed322a8973f83630b12715654c197dd392410400a5bfa1a73"}, + {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1414e8b124611bf4df8d77215bd32cba6e3425da8ce9c1f1046149615e3a9a31"}, + {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:93b16b08f94c92cab88073ffd185070cdcb29f1b98df8b28e6649145b7f2c90d"}, + {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fbc86ae8cc129c801e7baaafe3addf3c8d49c9c1597c44bdf2d78139707c3c62"}, + {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b5ba058610e8289a07db2a57bce45a1793ec0d3d11db28c047aae2aa1a832572"}, + {file = "coverage-6.3.3-cp37-cp37m-win32.whl", hash = "sha256:8329635c0781927a2c6ae068461e19674c564e05b86736ab8eb29c420ee7dc20"}, + {file = "coverage-6.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:e5af1feee71099ae2e3b086ec04f57f9950e1be9ecf6c420696fea7977b84738"}, + {file = "coverage-6.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e814a4a5a1d95223b08cdb0f4f57029e8eab22ffdbae2f97107aeef28554517e"}, + {file = "coverage-6.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:61f4fbf3633cb0713437291b8848634ea97f89c7e849c2be17a665611e433f53"}, + {file = "coverage-6.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3401b0d2ed9f726fadbfa35102e00d1b3547b73772a1de5508ef3bdbcb36afe7"}, + {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8586b177b4407f988731eb7f41967415b2197f35e2a6ee1a9b9b561f6323c8e9"}, + {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:892e7fe32191960da559a14536768a62e83e87bbb867e1b9c643e7e0fbce2579"}, + {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:afb03f981fadb5aed1ac6e3dd34f0488e1a0875623d557b6fad09b97a942b38a"}, + {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cbe91bc84be4e5ef0b1480d15c7b18e29c73bdfa33e07d3725da7d18e1b0aff2"}, + {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:91502bf27cbd5c83c95cfea291ef387469f2387508645602e1ca0fd8a4ba7548"}, + {file = "coverage-6.3.3-cp38-cp38-win32.whl", hash = "sha256:c488db059848702aff30aa1d90ef87928d4e72e4f00717343800546fdbff0a94"}, + {file = "coverage-6.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6534fcdfb5c503affb6b1130db7b5bfc8a0f77fa34880146f7a5c117987d0"}, + {file = "coverage-6.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc692c9ee18f0dd3214843779ba6b275ee4bb9b9a5745ba64265bce911aefd1a"}, + {file = "coverage-6.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:462105283de203df8de58a68c1bb4ba2a8a164097c2379f664fa81d6baf94b81"}, + {file = "coverage-6.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc972d829ad5ef4d4c5fcabd2bbe2add84ce8236f64ba1c0c72185da3a273130"}, + {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:06f54765cdbce99901871d50fe9f41d58213f18e98b170a30ca34f47de7dd5e8"}, + {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7835f76a081787f0ca62a53504361b3869840a1620049b56d803a8cb3a9eeea3"}, + {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6f5fee77ec3384b934797f1873758f796dfb4f167e1296dc00f8b2e023ce6ee9"}, + {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:baa8be8aba3dd1e976e68677be68a960a633a6d44c325757aefaa4d66175050f"}, + {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d06380e777dd6b35ee936f333d55b53dc4a8271036ff884c909cf6e94be8b6c"}, + {file = "coverage-6.3.3-cp39-cp39-win32.whl", hash = "sha256:f8cabc5fd0091976ab7b020f5708335033e422de25e20ddf9416bdce2b7e07d8"}, + {file = "coverage-6.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c9441d57b0963cf8340268ad62fc83de61f1613034b79c2b1053046af0c5284"}, + {file = "coverage-6.3.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:d522f1dc49127eab0bfbba4e90fa068ecff0899bbf61bf4065c790ddd6c177fe"}, + {file = "coverage-6.3.3.tar.gz", hash = "sha256:2781c43bffbbec2b8867376d4d61916f5e9c4cc168232528562a61d1b4b01879"}, ] -coverage = [] deepdiff = [ {file = "deepdiff-5.8.1-py3-none-any.whl", hash = "sha256:e9aea49733f34fab9a0897038d8f26f9d94a97db1790f1b814cced89e9e0d2b7"}, {file = "deepdiff-5.8.1.tar.gz", hash = "sha256:8d4eb2c4e6cbc80b811266419cb71dd95a157094a3947ccf937a94d44943c7b8"}, ] dill = [ - {file = "dill-0.3.5.1-py2.py3-none-any.whl", hash = "sha256:33501d03270bbe410c72639b350e941882a8b0fd55357580fbc873fba0c59302"}, - {file = "dill-0.3.5.1.tar.gz", hash = "sha256:d75e41f3eff1eee599d738e76ba8f4ad98ea229db8b085318aa2b3333a208c86"}, + {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"}, + {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"}, +] +distlib = [ + {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, + {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, +] +docutils = [ + {file = "docutils-0.17.1-py2.py3-none-any.whl", hash = "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61"}, + {file = "docutils-0.17.1.tar.gz", hash = "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125"}, +] +filelock = [ + {file = "filelock-3.7.0-py3-none-any.whl", hash = "sha256:c7b5fdb219b398a5b28c8e4c1893ef5f98ece6a38c6ab2c22e26ec161556fed6"}, + {file = "filelock-3.7.0.tar.gz", hash = "sha256:b795f1b42a61bbf8ec7113c341dad679d772567b936fbd1bf43c9a238e673e20"}, +] +flake8 = [ + {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"}, + {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"}, ] -distlib = [] -docutils = [] -filelock = [] -flake8 = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -imagesize = [] +imagesize = [ + {file = "imagesize-1.3.0-py2.py3-none-any.whl", hash = "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c"}, + {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, +] importlib-metadata = [ - {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, - {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, + {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, + {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -955,7 +991,10 @@ markupsafe = [ {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] -mccabe = [] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, @@ -968,29 +1007,9 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pathspec = [] -pendulum = [ - {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, - {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, - {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, - {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, - {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, - {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, - {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, - {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, - {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, - {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, - {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +pathspec = [ + {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, + {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, @@ -1004,25 +1023,38 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pycodestyle = [] -pyflakes = [] -pygments = [] -pylint = [] +pycodestyle = [ + {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, + {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, +] +pyflakes = [ + {file = "pyflakes-2.4.0-py2.py3-none-any.whl", hash = "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"}, + {file = "pyflakes-2.4.0.tar.gz", hash = "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c"}, +] +pygments = [ + {file = "Pygments-2.12.0-py3-none-any.whl", hash = "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"}, + {file = "Pygments-2.12.0.tar.gz", hash = "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb"}, +] +pylint = [ + {file = "pylint-2.13.9-py3-none-any.whl", hash = "sha256:705c620d388035bdd9ff8b44c5bcdd235bfb49d276d488dd2c8ff1736aa42526"}, + {file = "pylint-2.13.9.tar.gz", hash = "sha256:095567c96e19e6f57b5b907e67d265ff535e588fe26b12b5ebe1fc5645b2c731"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] -pytest = [] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +pytest = [ + {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, + {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, +] +pytz = [ + {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, + {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, ] -pytz = [] -pytzdata = [ - {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, - {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +requests = [ + {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, + {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, ] -requests = [] requirements-parser = [ {file = "requirements-parser-0.5.0.tar.gz", hash = "sha256:3336f3a3ae23e06d3f0f88595e4052396e3adf91688787f637e5d2ca1a904069"}, {file = "requirements_parser-0.5.0-py3-none-any.whl", hash = "sha256:e7fcdcd04f2049e73a9fb150d8a0f9d51ce4108f5f7cbeac74c484e17b12bcd9"}, @@ -1070,7 +1102,10 @@ snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] -sphinx = [] +sphinx = [ + {file = "Sphinx-4.5.0-py3-none-any.whl", hash = "sha256:ebf612653238bcc8f4359627a9b7ce44ede6fdd75d9d30f68255c7383d3a6226"}, + {file = "Sphinx-4.5.0.tar.gz", hash = "sha256:7bf8ca9637a4ee15af412d1a1d9689fec70523a68ca9bb9127c2f3eeb344e2e6"}, +] sphinxcontrib-applehelp = [ {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"}, @@ -1103,12 +1138,26 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -tomlkit = [] -tox = [] -types-setuptools = [] -typing-extensions = [] -urllib3 = [] -virtualenv = [] +tox = [ + {file = "tox-3.25.0-py2.py3-none-any.whl", hash = "sha256:0805727eb4d6b049de304977dfc9ce315a1938e6619c3ab9f38682bb04662a5a"}, + {file = "tox-3.25.0.tar.gz", hash = "sha256:37888f3092aa4e9f835fc8cc6dadbaaa0782651c41ef359e3a5743fcb0308160"}, +] +types-setuptools = [ + {file = "types-setuptools-57.4.14.tar.gz", hash = "sha256:df02fe1dd244f58cf4e67cfc3d0a97930a2d61a72dd89f21d81c71017cd83f9a"}, + {file = "types_setuptools-57.4.14-py3-none-any.whl", hash = "sha256:828f7e7e51e157876f47c80518b23ba0c3c36aa8081efd39d5d39f393938aec9"}, +] +typing-extensions = [ + {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, + {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, +] +urllib3 = [ + {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, + {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, +] +virtualenv = [ + {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, + {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, +] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, @@ -1179,4 +1228,7 @@ yapf = [ {file = "yapf-0.32.0-py2.py3-none-any.whl", hash = "sha256:8fea849025584e486fd06d6ba2bed717f396080fd3cc236ba10cb97c4c51cf32"}, {file = "yapf-0.32.0.tar.gz", hash = "sha256:a3f5085d37ef7e3e004c4ba9f9b3e40c54ff1901cd111f05145ae313a7c67d1b"}, ] -zipp = [] +zipp = [ + {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, + {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, +] From d0efeb296f12eb182c203ba7ebbe9560fe8f1cd5 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis <95035433+yannis-arg@users.noreply.github.com> Date: Thu, 22 Sep 2022 16:08:23 +0200 Subject: [PATCH 21/23] new: usr: Update documentation for FLYCS-547,868,540,814 features (#44) * new: usr: Update documentation for FLYCS-547,868,540,814 features * add poetry lock cmd in test workflow * correct test_to_dict_env test --- .github/workflows/test.yml | 1 + docs/custom_operators.rst | 39 ++++++++++++++++++ docs/index.rst | 1 + docs/transformations.rst | 81 ++++++++++++++++++++++++++++++++++++++ docs/triggers.rst | 80 ++++++++++++++++++++++++++++++++++++- docs/usage.rst | 44 +++++++++++++++++++++ tests/test_pipeline.py | 3 +- 7 files changed, 246 insertions(+), 3 deletions(-) create mode 100644 docs/transformations.rst diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5307bb1..e2a215e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install poetry + poetry lock poetry install - name: Check Format run: poetry run invoke format --check diff --git a/docs/custom_operators.rst b/docs/custom_operators.rst index 2a7a775..c5a6b33 100644 --- a/docs/custom_operators.rst +++ b/docs/custom_operators.rst @@ -10,3 +10,42 @@ Then you link this function to a CustomCode object. .. literalinclude:: examples/custom_operator.py :language: python + +How to use Custom arguments inside builder function? +####################################################### + +It is possible to use kwargs dict arguments inside the builder function of the Custom Operator. + +This feature allows you to create specific airflow Operators based on arguments value or the environment the DAG is running. +To be able to access those argument, you must defined them using the func_kwargs attribute when creatin the CustomCode object. + +See the following example : + +.. code-block:: python + + def build(dag, env=None, task_type = "msg", **kwargs): + from airflow.operators.dummy import DummyOperator + if kwargs.get("key"): + return DummyOperator(task_id=f"dummy_{env}_{task_type}_{kwargs['key']}",dag=dag) + else : + return DummyOperator(task_id=f'dummy_{env}_{task_type}',dag=dag) + + + mycode = CustomCode( + name="dummy_env_custom", + version="1.0.0", + operator_builder=build, + requirements=[], + func_kwargs={"task_type": "msg"}, + ) + mycode_kwargs = CustomCode( + name="dummy_env_custom_kwargs", + version="1.0.0", + operator_builder=build, + requirements=[], + func_kwargs={"task_type": "msg","key":"random_string"}, + ) + +Two CustomCode objects are created in this example. The first one defines only one argument (task_type) the result of the builder function will render a DummyOperator named based on the environment the DAG is running and based on the task_type argument value. + +The second CustomCode object, add the argument key with the value msg. Because this second argument is not already defined in the builder function, you must include the kwargs dictionnary argument in the builder function and access the key argument via the kwargs.get(“key”) function. diff --git a/docs/index.rst b/docs/index.rst index dc18a20..8496d92 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,6 +8,7 @@ Welcome to Flycs SDK's documentation! readme installation usage + transformations views functions stored_procedures diff --git a/docs/transformations.rst b/docs/transformations.rst new file mode 100644 index 0000000..dc17fb2 --- /dev/null +++ b/docs/transformations.rst @@ -0,0 +1,81 @@ +===== +Transformations +===== + +Flycs lets you define BigQuery queries that will be run as BigQuery Operators during the DAG run in which the query is used. +To define a query using the python SDK you must use a Transformation object. + +.. code-block:: python + + from flycs_sdk.transformations import Transformation + query = Transformation( + name="simple_copy", + query="SELECT * FROM raw.alpha.employees.employees AS raw", + version="1.0.0", + static=True, + ) + +How to use the Flycs SDK to create parameterized queries ? +####################################################### + +It can happen that you need to generate some queries with information that are dynamic, or you want to create a pipeline that uses the same query but on different table. +To do all of these things, the easiest way is to leverage the power of the Flycs SDK. + +In the following example we will show how you can define a pipeline that contains parameterized queries. + +.. code-block:: python + + from datetime import datetime, timezone + + from flycs_sdk.pipelines import Pipeline, PipelineKind + from flycs_sdk.entities import Entity + from flycs_sdk.transformations import Transformation + + # define your list of parameters + parameters = [ + ("table1", "tables2"), + ("table3", "tables4"), + ("table5", "tables6"), + ] + + # generate the transformation for each parameter + transformations = [] + for table1, table2 in parameters: + query = Transformation( + name="transformation_" + "_".join([table1, table2]), + # Notice how we generate the content of the query using + # the parameters define at the top of the file + query=f"SELECT * FROM {table1} LEFT JOIN {table2}", + version="1.0.0", + ) + transformations.append(query) + + # define the entity + entity = Entity( + name="my_entity", + version="1.0.0", + stage_config={}, + transformations={}, + ) + # insert the transformations into the entity + for t in transformations: + # stage_config is a dict that contains the stage name as key and a dictionnary of + # transformation name and version as value + entity.stage_config["staging"] = {t.name: t.version for t in transformations} + # stage_config is a dict that contains the stage name as key and a dictionnary of + # transformation name and transformation object as value + entity.transformations["staging"] = {t.name: t for t in transformations} + + # define the pipeline + my_pipeline = Pipeline( + name="my_pipeline", + version="1.0.0", + schedule="10 10 * * *", + entities=[entity], + kind=PipelineKind.VANILLA, + start_time=datetime.now(tz=timezone.utc), + ) + + # to be picked up by the framework, all the pipelines needs to be + # added to a variables called `pipelines` + pipelines = [my_pipeline] diff --git a/docs/triggers.rst b/docs/triggers.rst index 9a18a91..8fdf6d6 100644 --- a/docs/triggers.rst +++ b/docs/triggers.rst @@ -8,13 +8,14 @@ The supported type of triggers are: - `PubSub topic` - `Google Cloud Storage` - `Other pipeline` +- `Multiple Other pipelines` -PubSub topic +`PubSub topic` ############ This triggers creates a subscription to the topic and then waits for any message to come. One a message is received, the rest of the pipeline is executed. -The PubSub trigger has 2 property you can configure: +The PubSub trigger has 2 properties you can configure: - **topic**: The full path to a pubsub topic. The topic MUST exist before the pipeline with the trigger is executed. - **subscription_project**: Optional property that let you choose in which project the subscription to the topic will be created. @@ -187,3 +188,78 @@ Same example with the python SDK: start_time=datetime.now(tz=timezone.utc) schedule=master, # Here we pass the master Pipeline object directly into the `schedule` field. ) + +`Multiple Other pipelines` +################ + +This trigger is used when you want a child pipeline to be triggered by multiple parents pipelines. Using this trigger will actually used sensor on the child pipeline to wait for all the parents pipelines to finish. + +This feature use ExternalSensor on the child pipeline. The schedule period of it will be automatically computed to run at the wider cron job scheduled of the list of the parents pipelines. + +E.g : Having pipeline_A running 30 9 * * * (every day UTC at 9.30) and pipeline_B running \*/30 * * * * (every 30 minutes) with a child pipeline_C depending on both A and B. Then, the pipeline_C will be scheduled with the cron configuration 30 9 * * \*. + +Using yaml definition : + +.. code-block:: yaml + + # parent pipeline_A + name: pipeline_A + kind: vanilla + version: 1.0.0 + entities: + ... # removed for brevity + schedule: "30 9 * * *" + start_time: "2021-01-01T00:00:00" + + # parent pipeline_B + name: pipeline_B + kind: vanilla + version: 1.0.0 + entities: + ... # removed for brevity + schedule: "*/30 * * * *" + start_time: "2021-01-01T00:00:00" + + # child pipeline_C + name: pipeline_C + kind: vanilla + version: 1.0.0 + entities: + ... # removed for brevity + schedule: ["pipeline_A","pipeline_B"] + start_time: "2021-01-01T00:00:00" # /!\ start_time of all the parents pipeline (A & B) must be the same as the pipeline_C + + +Using python pdk : + + +.. code-block:: python + + now_datetime = datetime.now(tz=timezone.utc) + + pipeline_A = Pipeline( + name="pipeline_A", + version="1.0.0", + entities=[entity], + kind=PipelineKind.VANILLA, + start_time=now_datetime, + schedule="30 9 * * *", + ) + + pipeline_B = Pipeline( + name="pipeline_B", + version="1.0.0", + entities=[entity], + kind=PipelineKind.VANILLA, + start_time=now_datetime, + schedule="*/30 * * * *", + ) + + pipeline_C = Pipeline( + name="pipeline_C", + version="1.0.0", + entities=[entity], + kind=PipelineKind.VANILLA, + start_time=now_datetime, + schedule=[pipeline_A, pipeline_B], # Here we pass the list of parents pipelines objects. + ) diff --git a/docs/usage.rst b/docs/usage.rst index 6c8d8d4..a929fb7 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -93,3 +93,47 @@ Parameterized pipeline and entity also allow to introduce custom logic. Here is .. literalinclude:: examples/parametrize_pipeline.py :language: python + +How to set up different schedule based on the environment the DAG is running ? +################################################################## +You can schedule your pipeline with a different cron job syntax based on the environment it will be running. The environment available are the following : **sbx** (sandbox), **tst** (test), **acc** (acceptance), **prd** (production). +To define the schedule for each environment , you must use a dictionary object : + +.. code-block:: python + + import pendulum + + from flycs_sdk.pipelines import Pipeline, PipelineKind + from .entities import entity_demo + + demo_python = Pipeline( + name="demo_python_env_scheduled", + version="1.0.0", + schedule={"sbx":"30 10 * * *", "tst":"@daily" , "acc":"@daily" , "prd" : "@weekly"}, + entities=[entity_demo], + kind=PipelineKind.VANILLA, + start_time = pendulum.now(tz="Europe/Brussels") + ) + +How to make a Timezone aware DAG ? +####################################################### +To make your DAG pipeline running in a specific timezone you first need to make sure you pick a timezone name defined in `this Timezone list `_. +Using the python-sdk you have to use the pendulum package library : + +.. code-block:: python + + import pendulum + + from flycs_sdk.pipelines import Pipeline, PipelineKind + from .entities import entity_demo + + demo_python = Pipeline( + name="demo_python", + version="1.0.0", + schedule="10 10 * * *", + entities=[entity_demo], + kind=PipelineKind.VANILLA, + start_time = pendulum.now(tz="Europe/Brussels") + ) + +NB : By default, using a datetime object for python Pipeline instead of a pendulum object will consider the DAG is running in UTC timezone. diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d695d56..a07f05f 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -224,7 +224,8 @@ def test_to_dict_env(self, my_pipeline_env, my_entity): "version": pipeline_version, "schedule": env_schedule, "kind": pipeline_kind.value, - "start_time": "2020-12-02T15:38:34+0000", + "start_time": "2020-12-02T15:38:34+00:00", + "timezone": "UTC", "trigger": None, "params": {}, "entities": [ From 23aca9ccdfc9c6ec3361567362fb73247a3f3c80 Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis <95035433+yannis-arg@users.noreply.github.com> Date: Tue, 27 Sep 2022 13:50:04 +0200 Subject: [PATCH 22/23] fix: dev: Add the keyset_used attribute to view & correct test_view class [FLYCS-622] (#45) * fix: dev: add keyset_used on View object [FLYCS-622] * correct test_to_dict View obj --- flycs_sdk/views.py | 7 +++++++ tests/test_view.py | 1 + 2 files changed, 8 insertions(+) diff --git a/flycs_sdk/views.py b/flycs_sdk/views.py index 4d7f79f..a144d57 100644 --- a/flycs_sdk/views.py +++ b/flycs_sdk/views.py @@ -22,6 +22,7 @@ def __init__( destination_data_mart: Optional[str] = None, force_cache_refresh: Optional[bool] = False, schema: Optional[List[FieldConfig]] = None, + keysets_used: Optional[List[str]] = None, ): """Create a View object. @@ -37,6 +38,8 @@ def __init__( :type encrypt: Optional[bool] :param force_cache_refresh: whether or not we need to use the cache in the pii service :type force_cache_refresh: bool, optional + :param keysets_used: List of keysets used in the transformation + :type keysets_used: List[str] """ super().__init__( name=name, @@ -52,6 +55,7 @@ def __init__( self.dependencies = [] self.parsing_dependencies = [] self.force_cache_refresh = force_cache_refresh + self.keysets_used = keysets_used or [] @classmethod def from_dict(cls, d: dict): @@ -78,6 +82,7 @@ def from_dict(cls, d: dict): Dependency.from_dict(x) for x in d.get("PARSING_DEPENDS_ON") or [] ] view.force_cache_refresh = d.get("FORCE_CACHE_REFRESH", False) + view.keysets_used = d.get("KEYSETS_USED", []) return view def to_dict(self) -> dict: @@ -101,6 +106,7 @@ def to_dict(self) -> dict: "PARSING_DEPENDS_ON": [d.to_dict() for d in self.parsing_dependencies], "FORCE_CACHE_REFRESH": self.force_cache_refresh, "SCHEMA": [config.to_dict() for config in self.schema], + "KEYSETS_USED": self.keysets_used, } def __eq__(self, o) -> bool: @@ -119,4 +125,5 @@ def __eq__(self, o) -> bool: and self.parsing_dependencies == o.parsing_dependencies and self.force_cache_refresh == o.force_cache_refresh and self.schema == o.schema + and self.keysets_used == o.keysets_used ) diff --git a/tests/test_view.py b/tests/test_view.py index 30d0dea..0698950 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -42,6 +42,7 @@ def test_to_dict(self, my_view: View): "DEPENDS_ON": [], "PARSING_DEPENDS_ON": [], "FORCE_CACHE_REFRESH": view_force_cache_refresh, + "KEYSETS_USED": [], "SCHEMA": [], } From e055213155f6f89f962197367985ad4f0ea73e9d Mon Sep 17 00:00:00 2001 From: Yannis Argyrakis Date: Tue, 14 Feb 2023 16:21:16 +0100 Subject: [PATCH 23/23] =?UTF-8?q?Bump=20version:=200.12.0=20=E2=86=92=200.?= =?UTF-8?q?12.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- flycs_sdk/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 84ec897..2b84b6d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.12.0 +current_version = 0.12.1 commit = True tag = True diff --git a/flycs_sdk/__init__.py b/flycs_sdk/__init__.py index 1f4d285..f0fb9d3 100644 --- a/flycs_sdk/__init__.py +++ b/flycs_sdk/__init__.py @@ -2,4 +2,4 @@ __author__ = """Tristan Van Thielen""" __email__ = "tristan.van.thielen@devoteam.com" -__version__ = "0.12.0" +__version__ = "0.12.1" diff --git a/pyproject.toml b/pyproject.toml index 9050a30..1a81bbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ packages = [ {include = "tests", format = "sdist"}, ] readme = "README.rst" -version = "0.12.0" +version = "0.12.1" [tool.poetry.dependencies] python = ">=3.8,<3.11"