Moves provider packages scripts to dev

The change #10806 made airflow works with implicit packages
when "airflow" got imported. This is a good change, however
it has some unforeseen consequences. The 'provider_packages'
script copy all the providers code for backports in order
to refactor them to the empty "airflow" directory in
provider_packages folder. The #10806 change turned that
empty folder in 'airflow' package because it was in the
same directory as the provider_packages scripts.

Moving the scripts to dev solves this problem.

.pre-commit-config.yaml

@@ -182,7 +182,7 @@ repos:
       - id: pyupgrade
         args: ["--py36-plus"]
   - repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.6.0
+    rev: v1.7.0
     hooks:
       - id: rst-backticks
       - id: python-no-log-warn
@@ -200,7 +200,8 @@ repos:
     hooks:
       - id: isort
         name: Run isort to sort imports
-        types: [python]
+        args: [--filter-files, --profile, black]
+        files: \.py$
         # To keep consistent with the global isort skip config defined in setup.cfg
         exclude: ^build/.*$|^.tox/.*$|^venv/.*$
   - repo: https://github.com/pycqa/pydocstyle
@@ -336,7 +337,7 @@ repos:
           ^airflow/operators/.*$|
           ^airflow/sensors/.*$|
           ^airflow/providers/.*$|
-          ^provider_packages/.*$
+          ^dev/provider_packages/.*$
       - id: base-operator
         language: pygrep
         name: Make sure BaseOperator[Link] is imported from airflow.models outside of core

airflow/providers/amazon/backport_provider_setup.cfg

@@ -48,4 +48,4 @@ default_section = THIRDPARTY
 include_trailing_comma = true
 known_first_party=airflow,tests
 multi_line_output=5
-profile = "black"
+profile = black

airflow/providers/amazon/backport_provider_setup.py

@@ -31,7 +31,7 @@
 import sys
 from os.path import dirname
 
-from setuptools import find_packages, setup
+from setuptools import find_namespace_packages, setup
 
 logger = logging.getLogger(__name__)
 
@@ -58,7 +58,7 @@ def do_setup(version_suffix_for_pypi=''):
         long_description_content_type='text/markdown',
         license='Apache License 2.0',
         version=version + version_suffix_for_pypi,
-        packages=find_packages(include=['airflow.providers.amazon*']),
+        packages=find_namespace_packages(include=['airflow.providers.amazon', 'airflow.providers.amazon.*']),
         zip_safe=False,
         install_requires=['apache-airflow~=1.10', 'boto3>=1.12.0,<2.0.0', 'watchtower~=0.7.3'],
         setup_requires=['setuptools', 'wheel'],

airflow/providers/amazon/setup.cfg

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[metadata]
+name = Airflow Provider: apache-airflow-providers-amazon
+summary = Provider for Apache Airflow. Implements apache-airflow-providers-amazon package
+description-file = README.md
+author = Apache Airflow PMC
+author-email = [email protected]
+license = Apache License, Version 2.0
+license_files =
+   LICENSE
+   NOTICE
+
+[bdist_wheel]
+python-tag=py3
+
+[files]
+packages = airflow.providers.amazon
+
+[easy_install]
+
+[mypy]
+ignore_missing_imports = True
+no_implicit_optional = True
+warn_redundant_casts = True
+warn_unused_ignores = False
+pretty = True
+
+[isort]
+line_length=110
+combine_as_imports = true
+default_section = THIRDPARTY
+include_trailing_comma = true
+known_first_party=airflow,tests
+multi_line_output=5
+profile = black

airflow/providers/amazon/setup.py

@@ -0,0 +1,109 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE\
+# OVERWRITTEN WHEN RUNNING
+#
+# ./breeze prepare-provider-readme
+#
+# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
+# `SETUP_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY
+
+"""Setup.py for the apache-airflow-providers-amazon package."""
+
+import logging
+import os
+import sys
+from os.path import dirname
+
+from setuptools import find_namespace_packages, setup
+
+logger = logging.getLogger(__name__)
+
+version = '0.0.2a1'
+
+my_dir = dirname(__file__)
+
+try:
+    with open(os.path.join(my_dir, 'airflow/providers/amazon/README.md'), encoding='utf-8') as f:
+        long_description = f.read()
+except FileNotFoundError:
+    long_description = ''
+
+
+def do_setup(version_suffix_for_pypi=''):
+    """Perform the package apache-airflow-providers-amazon setup."""
+    setup(
+        name='apache-airflow-providers-amazon',
+        description='Provider package ' 'apache-airflow-providers-amazon for Apache Airflow',
+        long_description=long_description,
+        long_description_content_type='text/markdown',
+        license='Apache License 2.0',
+        version=version + version_suffix_for_pypi,
+        packages=find_namespace_packages(include=['airflow.providers.amazon', 'airflow.providers.amazon.*']),
+        zip_safe=False,
+        install_requires=['apache-airflow>=2.0.0a0', 'boto3>=1.12.0,<2.0.0', 'watchtower~=0.7.3'],
+        setup_requires=['setuptools', 'wheel'],
+        extras_require={
+            'apache.hive': ['apache-airflow-providers-apache-hive'],
+            'google': ['apache-airflow-providers-google'],
+            'imap': ['apache-airflow-providers-imap'],
+            'mongo': ['apache-airflow-providers-mongo'],
+            'mysql': ['apache-airflow-providers-mysql'],
+            'postgres': ['apache-airflow-providers-postgres'],
+            'ssh': ['apache-airflow-providers-ssh'],
+        },
+        classifiers=[
+            'Development Status :: 5 - Production/Stable',
+            'Environment :: Console',
+            'Environment :: Web Environment',
+            'Intended Audience :: Developers',
+            'Intended Audience :: System Administrators',
+            'License :: OSI Approved :: Apache Software License',
+            'Programming Language :: Python :: 3.6',
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Topic :: System :: Monitoring',
+        ],
+        author='Apache Software Foundation',
+        author_email='[email protected]',
+        url='http://airflow.apache.org/',
+        download_url=('https://archive.apache.org/dist/airflow/providers'),
+        python_requires='~=3.6',
+        project_urls={
+            'Documentation': 'https://airflow.apache.org/docs/',
+            'Bug Tracker': 'https://github.com/apache/airflow/issues',
+            'Source Code': 'https://github.com/apache/airflow',
+        },
+    )
+
+
+#
+# Note that --version-suffix-for-pypi should only be used in case we generate RC packages for PyPI
+# Those packages should have actual RC version in order to be published even if source version
+# should be the final one.
+#
+if __name__ == "__main__":
+    suffix = ''
+    if len(sys.argv) > 1 and sys.argv[1] == "--version-suffix-for-pypi":
+        if len(sys.argv) < 3:
+            print("ERROR! --version-suffix-for-pypi needs parameter!", file=sys.stderr)
+            sys.exit(1)
+        suffix = sys.argv[2]
+        sys.argv = [sys.argv[0]] + sys.argv[3:]
+    do_setup(version_suffix_for_pypi=suffix)

airflow/providers/apache/cassandra/backport_provider_setup.cfg

@@ -48,4 +48,4 @@ default_section = THIRDPARTY
 include_trailing_comma = true
 known_first_party=airflow,tests
 multi_line_output=5
-profile = "black"
+profile = black

airflow/providers/apache/cassandra/backport_provider_setup.py

@@ -31,7 +31,7 @@
 import sys
 from os.path import dirname
 
-from setuptools import find_packages, setup
+from setuptools import find_namespace_packages, setup
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +59,9 @@ def do_setup(version_suffix_for_pypi=''):
         long_description_content_type='text/markdown',
         license='Apache License 2.0',
         version=version + version_suffix_for_pypi,
-        packages=find_packages(include=['airflow.providers.apache.cassandra*']),
+        packages=find_namespace_packages(
+            include=['airflow.providers.apache.cassandra', 'airflow.providers.apache.cassandra.*']
+        ),
         zip_safe=False,
         install_requires=['apache-airflow~=1.10', 'cassandra-driver>=3.13.0,<3.21.0'],
         setup_requires=['setuptools', 'wheel'],

airflow/providers/apache/cassandra/setup.cfg

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[metadata]
+name = Airflow Provider: apache-airflow-providers-apache-cassandra
+summary = Provider for Apache Airflow. Implements apache-airflow-providers-apache-cassandra package
+description-file = README.md
+author = Apache Airflow PMC
+author-email = [email protected]
+license = Apache License, Version 2.0
+license_files =
+   LICENSE
+   NOTICE
+
+[bdist_wheel]
+python-tag=py3
+
+[files]
+packages = airflow.providers.apache.cassandra
+
+[easy_install]
+
+[mypy]
+ignore_missing_imports = True
+no_implicit_optional = True
+warn_redundant_casts = True
+warn_unused_ignores = False
+pretty = True
+
+[isort]
+line_length=110
+combine_as_imports = true
+default_section = THIRDPARTY
+include_trailing_comma = true
+known_first_party=airflow,tests
+multi_line_output=5
+profile = black

airflow/providers/apache/cassandra/setup.py

@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE\
+# OVERWRITTEN WHEN RUNNING
+#
+# ./breeze prepare-provider-readme
+#
+# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
+# `SETUP_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY
+
+"""Setup.py for the apache-airflow-providers-apache-cassandra package."""
+
+import logging
+import os
+import sys
+from os.path import dirname
+
+from setuptools import find_namespace_packages, setup
+
+logger = logging.getLogger(__name__)
+
+version = '0.0.2a1'
+
+my_dir = dirname(__file__)
+
+try:
+    with open(os.path.join(my_dir, 'airflow/providers/apache/cassandra/README.md'), encoding='utf-8') as f:
+        long_description = f.read()
+except FileNotFoundError:
+    long_description = ''
+
+
+def do_setup(version_suffix_for_pypi=''):
+    """Perform the package apache-airflow-providers-apache-cassandra setup."""
+    setup(
+        name='apache-airflow-providers-apache-cassandra',
+        description='Provider package ' 'apache-airflow-providers-apache-cassandra for Apache Airflow',
+        long_description=long_description,
+        long_description_content_type='text/markdown',
+        license='Apache License 2.0',
+        version=version + version_suffix_for_pypi,
+        packages=find_namespace_packages(
+            include=['airflow.providers.apache.cassandra', 'airflow.providers.apache.cassandra.*']
+        ),
+        zip_safe=False,
+        install_requires=['apache-airflow>=2.0.0a0', 'cassandra-driver>=3.13.0,<3.21.0'],
+        setup_requires=['setuptools', 'wheel'],
+        extras_require={},
+        classifiers=[
+            'Development Status :: 5 - Production/Stable',
+            'Environment :: Console',
+            'Environment :: Web Environment',
+            'Intended Audience :: Developers',
+            'Intended Audience :: System Administrators',
+            'License :: OSI Approved :: Apache Software License',
+            'Programming Language :: Python :: 3.6',
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Topic :: System :: Monitoring',
+        ],
+        author='Apache Software Foundation',
+        author_email='[email protected]',
+        url='http://airflow.apache.org/',
+        download_url=('https://archive.apache.org/dist/airflow/providers'),
+        python_requires='~=3.6',
+        project_urls={
+            'Documentation': 'https://airflow.apache.org/docs/',
+            'Bug Tracker': 'https://github.com/apache/airflow/issues',
+            'Source Code': 'https://github.com/apache/airflow',
+        },
+    )
+
+
+#
+# Note that --version-suffix-for-pypi should only be used in case we generate RC packages for PyPI
+# Those packages should have actual RC version in order to be published even if source version
+# should be the final one.
+#
+if __name__ == "__main__":
+    suffix = ''
+    if len(sys.argv) > 1 and sys.argv[1] == "--version-suffix-for-pypi":
+        if len(sys.argv) < 3:
+            print("ERROR! --version-suffix-for-pypi needs parameter!", file=sys.stderr)
+            sys.exit(1)
+        suffix = sys.argv[2]
+        sys.argv = [sys.argv[0]] + sys.argv[3:]
+    do_setup(version_suffix_for_pypi=suffix)