From b497234eda4b4bdfda0b09e3674012c6ebebaf2b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 31 Aug 2023 07:35:28 +0200 Subject: [PATCH] Reorganize devel_only extra in airflow's setup.py (#33907) The `devel_only` extra puts together all dependencies that are needed for CI image and in order to run tests in local virtualenv for various test cases of ours - but they are not needed as dependencies of particular providers. They were a little "bag of everything" and they were hiding some unused dependencies or dependencies that were either unused or they were actually provider dependencies already. For example we had qds-sdk dependency there which was really the qubole provider dependency and it held us back from removing deprecated boto library from CI image (removed in #33889). This PR organizes the dependency a bit better: * split it to logical groups * removes some unused dependencies * moves "amazon" mypy dependency from providers to here At later stage we will move the provider ones into "[devel]" extras of the providers as part of provider decooupling, but this will require a bit more changes in CI image building and some documentation update for developers. This is an intermediate step to organize it better. --- airflow/providers/amazon/provider.yaml | 8 +-- generated/provider_dependencies.json | 4 -- setup.py | 93 ++++++++++++++++++-------- 3 files changed, 67 insertions(+), 38 deletions(-) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 5f731f96643c3..d31a4bc1bbaa1 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -74,12 +74,10 @@ dependencies: # set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS + # NOTE!!! Make sure to update _MIN_BOTO3_VERSION in setup.py when you update it here - boto3>=1.28.0 - - mypy-boto3-rds>=1.28.0 - - mypy-boto3-redshift-data>=1.28.0 - - mypy-boto3-s3>=1.28.0 - - mypy-boto3-appflow>=1.28.0 - # NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS + # NOTE!!! BOTOCORE version is always shifted by 3 MINOR VERSIONS from boto3 + # See https://github.com/boto/boto3/issues/2702 - botocore>=1.31.0 - asgiref # watchtower 3 has been released end Jan and introduced breaking change across the board that might diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 8098cdd629402..014a738668893 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -29,10 +29,6 @@ "boto3>=1.28.0", "botocore>=1.31.0", "jsonpath_ng>=1.5.3", - "mypy-boto3-appflow>=1.28.0", - "mypy-boto3-rds>=1.28.0", - "mypy-boto3-redshift-data>=1.28.0", - "mypy-boto3-s3>=1.28.0", "redshift_connector>=2.0.888", "sqlalchemy_redshift>=0.8.6", "watchtower~=2.0.1" diff --git a/setup.py b/setup.py index db438112c8d8c..a2d9a0a969160 100644 --- a/setup.py +++ b/setup.py @@ -384,7 +384,6 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve # Make sure to upgrade the mypy version in update-common-sql-api-stubs in .pre-commit-config.yaml # when you upgrade it here !!!! "mypy==1.2.0", - "types-boto", "types-certifi", "types-croniter", "types-Deprecated", @@ -405,50 +404,86 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve "types-PyYAML", ] -# Dependencies needed for development only -devel_only = [ +# make sure to update providers/amazon/provider.yaml botocore min version when you update it here +_MIN_BOTO3_VERSION = "1.28.0" + +_devel_only_amazon = [ "aws_xray_sdk", - "beautifulsoup4>=4.7.1", - "black", - "blinker", - "bowler", - "click>=8.0", - "coverage>=7.2", + "moto[cloudformation, glue]>=4.0", + f"mypy-boto3-rds>={_MIN_BOTO3_VERSION}", + f"mypy-boto3-redshift-data>={_MIN_BOTO3_VERSION}", + f"mypy-boto3-s3>={_MIN_BOTO3_VERSION}", + f"mypy-boto3-appflow>={_MIN_BOTO3_VERSION}", +] + +_devel_only_azure = [ + "pywinrm", +] + +_devel_only_breeze = [ "filelock", - "gitpython", +] + +_devel_only_debuggers = [ "ipdb", - "jira", - "jsondiff", - "jsonpath_ng>=1.5.3", - "mongomock", - "moto[cloudformation, glue]>=4.0", - "paramiko", +] + +_devel_only_devscripts = [ + "click>=8.0", + "gitpython", "pipdeptree", - "pre-commit", - "pypsrp", "pygithub", + "rich-click>=1.5", + "semver", + "towncrier", + "twine", + "wheel", +] + +_devel_only_mongo = [ + "mongomock", +] + +_devel_only_sentry = [ + "blinker", +] + +_devel_only_static_checks = [ + "pre-commit", + "black", + "ruff>=0.0.219", + "yamllint", +] + +_devel_only_tests = [ + "aioresponses", + "beautifulsoup4>=4.7.1", + "coverage>=7.2", "pytest", "pytest-asyncio", "pytest-capture-warnings", "pytest-cov", + "pytest-httpx", "pytest-instafail", "pytest-mock", "pytest-rerunfailures", "pytest-timeouts", "pytest-xdist", - "python-jose", - "pywinrm", - "pytest-httpx", "requests_mock", - "rich-click>=1.5", - "ruff>=0.0.219", - "semver", "time-machine", - "towncrier", - "twine", - "wheel", - "yamllint", - "aioresponses", +] + +# Dependencies needed for development only +devel_only = [ + *_devel_only_amazon, + *_devel_only_azure, + *_devel_only_breeze, + *_devel_only_debuggers, + *_devel_only_devscripts, + *_devel_only_mongo, + *_devel_only_sentry, + *_devel_only_static_checks, + *_devel_only_tests, ] aiobotocore = [