Reorganize devel_only extra in airflow's setup.py (#33907)

The `devel_only` extra puts together all dependencies that are
needed for CI image and in order to run tests in local virtualenv
for various test cases of ours - but they are not needed as dependencies
of particular providers. They were a little "bag of everything"
and they were hiding some unused dependencies or dependencies that
were either unused or they were actually provider dependencies already.

For example we had qds-sdk dependency there which was really the
qubole provider dependency and it held us back from removing
deprecated boto library from CI image (removed in #33889).

This PR organizes the dependency a bit better:

* split it to logical groups
* removes some unused dependencies
* moves "amazon" mypy dependency from providers to here

At later stage we will move the provider ones into "[devel]" extras
of the providers as part of provider decooupling, but this
will require a bit more changes in CI image building and some
documentation update for developers. This is an intermediate step
to organize it better.

airflow/providers/amazon/provider.yaml

@@ -74,12 +74,10 @@ dependencies:
   # set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end
   # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version
   # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS
+  # NOTE!!! Make sure to update _MIN_BOTO3_VERSION in setup.py when you update it here
   - boto3>=1.28.0
-  - mypy-boto3-rds>=1.28.0
-  - mypy-boto3-redshift-data>=1.28.0
-  - mypy-boto3-s3>=1.28.0
-  - mypy-boto3-appflow>=1.28.0
-  # NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS
+  # NOTE!!! BOTOCORE version is always shifted by 3 MINOR VERSIONS from boto3
+  # See https://github.com/boto/boto3/issues/2702
   - botocore>=1.31.0
   - asgiref
   # watchtower 3 has been released end Jan and introduced breaking change across the board that might

generated/provider_dependencies.json

@@ -29,10 +29,6 @@
       "boto3>=1.28.0",
       "botocore>=1.31.0",
       "jsonpath_ng>=1.5.3",
-      "mypy-boto3-appflow>=1.28.0",
-      "mypy-boto3-rds>=1.28.0",
-      "mypy-boto3-redshift-data>=1.28.0",
-      "mypy-boto3-s3>=1.28.0",
       "redshift_connector>=2.0.888",
       "sqlalchemy_redshift>=0.8.6",
       "watchtower~=2.0.1"

setup.py

@@ -384,7 +384,6 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve
     # Make sure to upgrade the mypy version in update-common-sql-api-stubs in .pre-commit-config.yaml
     # when you upgrade it here !!!!
     "mypy==1.2.0",
-    "types-boto",
     "types-certifi",
     "types-croniter",
     "types-Deprecated",
@@ -405,50 +404,86 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve
     "types-PyYAML",
 ]
 
-# Dependencies needed for development only
-devel_only = [
+# make sure to update providers/amazon/provider.yaml botocore min version when you update it here
+_MIN_BOTO3_VERSION = "1.28.0"
+
+_devel_only_amazon = [
     "aws_xray_sdk",
-    "beautifulsoup4>=4.7.1",
-    "black",
-    "blinker",
-    "bowler",
-    "click>=8.0",
-    "coverage>=7.2",
+    "moto[cloudformation, glue]>=4.0",
+    f"mypy-boto3-rds>={_MIN_BOTO3_VERSION}",
+    f"mypy-boto3-redshift-data>={_MIN_BOTO3_VERSION}",
+    f"mypy-boto3-s3>={_MIN_BOTO3_VERSION}",
+    f"mypy-boto3-appflow>={_MIN_BOTO3_VERSION}",
+]
+
+_devel_only_azure = [
+    "pywinrm",
+]
+
+_devel_only_breeze = [
     "filelock",
-    "gitpython",
+]
+
+_devel_only_debuggers = [
     "ipdb",
-    "jira",
-    "jsondiff",
-    "jsonpath_ng>=1.5.3",
-    "mongomock",
-    "moto[cloudformation, glue]>=4.0",
-    "paramiko",
+]
+
+_devel_only_devscripts = [
+    "click>=8.0",
+    "gitpython",
     "pipdeptree",
-    "pre-commit",
-    "pypsrp",
     "pygithub",
+    "rich-click>=1.5",
+    "semver",
+    "towncrier",
+    "twine",
+    "wheel",
+]
+
+_devel_only_mongo = [
+    "mongomock",
+]
+
+_devel_only_sentry = [
+    "blinker",
+]
+
+_devel_only_static_checks = [
+    "pre-commit",
+    "black",
+    "ruff>=0.0.219",
+    "yamllint",
+]
+
+_devel_only_tests = [
+    "aioresponses",
+    "beautifulsoup4>=4.7.1",
+    "coverage>=7.2",
     "pytest",
     "pytest-asyncio",
     "pytest-capture-warnings",
     "pytest-cov",
+    "pytest-httpx",
     "pytest-instafail",
     "pytest-mock",
     "pytest-rerunfailures",
     "pytest-timeouts",
     "pytest-xdist",
-    "python-jose",
-    "pywinrm",
-    "pytest-httpx",
     "requests_mock",
-    "rich-click>=1.5",
-    "ruff>=0.0.219",
-    "semver",
     "time-machine",
-    "towncrier",
-    "twine",
-    "wheel",
-    "yamllint",
-    "aioresponses",
+]
+
+# Dependencies needed for development only
+devel_only = [
+    *_devel_only_amazon,
+    *_devel_only_azure,
+    *_devel_only_breeze,
+    *_devel_only_debuggers,
+    *_devel_only_devscripts,
+    *_devel_only_mongo,
+    *_devel_only_sentry,
+    *_devel_only_static_checks,
+    *_devel_only_tests,
 ]
 
 aiobotocore = [