diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..0641e6f267bb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/sdks" # Location of package manifests + schedule: + interval: "daily" + - package-ecosystem: "pip" + directory: "/sdks/python" # Location of package manifests + schedule: + interval: "daily" + - package-ecosystem: "gradle" + directory: "/" # Location of package manifests + schedule: + interval: "daily" diff --git a/sdks/python/setup.py b/sdks/python/setup.py index a02f9f9c339c..70cec6f00a86 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -120,121 +120,11 @@ def get_version(): except ImportError: cythonize = lambda *args, **kwargs: [] -REQUIRED_PACKAGES = [ - # Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1 - 'crcmod>=1.7,<2.0', - # dataclasses backport for python_version<3.7. No version bound because this - # is Python standard since Python 3.7 and each Python version is compatible - # with a specific dataclasses version. - 'dataclasses;python_version<"3.7"', - 'orjson<4.0', - # Dill doesn't have forwards-compatibility guarantees within minor version. - # Pickles created with a new version of dill may not unpickle using older - # version of dill. It is best to use the same version of dill on client and - # server, therefore list of allowed versions is very narrow. - # See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.1.1,<0.3.2', - 'cloudpickle>=2.0.0,<3', - 'fastavro>=0.23.6,<2', - 'grpcio>=1.29.0,<2', - 'hdfs>=2.1.0,<3.0.0', - 'httplib2>=0.8,<0.21.0', - 'numpy>=1.14.3,<1.23.0', - 'pymongo>=3.8.0,<4.0.0', - 'protobuf>=3.12.2,<4', - 'proto-plus>=1.7.1,<2', - 'pyarrow>=0.15.1,<8.0.0', - 'pydot>=1.2.0,<2', - 'python-dateutil>=2.8.0,<3', - 'pytz>=2018.3', - 'requests>=2.24.0,<3.0.0', - 'typing-extensions>=3.7.0', -] - # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms. if sys.platform == 'win32' and sys.maxsize <= 2**32: - REQUIRED_PACKAGES = [ - p for p in REQUIRED_PACKAGES if not p.startswith('pyarrow') - ] - -REQUIRED_TEST_PACKAGES = [ - 'freezegun>=0.3.12', - 'joblib>=1.0.1', - 'mock>=1.0.1,<3.0.0', - 'pandas<2.0.0', - 'parameterized>=0.7.1,<0.8.0', - 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', - 'pyyaml>=3.12,<7.0.0', - 'requests_mock>=1.7,<2.0', - 'tenacity>=5.0.2,<6.0', - 'pytest>=4.4.0,<5.0', - 'pytest-xdist>=1.29.0,<2', - 'pytest-timeout>=1.3.3,<2', - 'scikit-learn>=0.20.0', - 'sqlalchemy>=1.3,<2.0', - 'psycopg2-binary>=2.8.5,<3.0.0', - 'testcontainers[mysql]>=3.0.3,<4.0.0', - 'cryptography>=36.0.0', -] - -GCP_REQUIREMENTS = [ - 'cachetools>=3.1.0,<5', - 'google-apitools>=0.5.31,<0.5.32', - # NOTE: Maintainers, please do not require google-auth>=2.x.x - # Until this issue is closed - # https://github.com/googleapis/google-cloud-python/issues/10566 - 'google-auth>=1.18.0,<3', - 'google-auth-httplib2>=0.1.0,<0.2.0', - 'google-cloud-datastore>=1.8.0,<2', - 'google-cloud-pubsub>=2.1.0,<3', - 'google-cloud-pubsublite>=1.2.0,<2', - # GCP packages required by tests - 'google-cloud-bigquery>=1.6.0,<3', - 'google-cloud-bigquery-storage>=2.6.3', - 'google-cloud-core>=0.28.1,<2', - 'google-cloud-bigtable>=0.31.1,<2', - 'google-cloud-spanner>=1.13.0,<2', - 'grpcio-gcp>=0.2.2,<1', - # GCP Packages required by ML functionality - 'google-cloud-dlp>=3.0.0,<4', - 'google-cloud-language>=1.3.0,<2', - 'google-cloud-videointelligence>=1.8.0,<2', - 'google-cloud-vision>=0.38.0,<2', - 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' -] - -INTERACTIVE_BEAM = [ - 'facets-overview>=1.0.0,<2', - 'google-cloud-dataproc>=3.0.0,<3.2.0', - # IPython>=8 is not compatible with Python<=3.7 - 'ipython>=7,<8;python_version<="3.7"', - 'ipython>=8,<9;python_version>"3.7"', - 'ipykernel>=6,<7', - 'ipywidgets>=7.6.5,<8', - # Skip version 6.1.13 due to - # https://github.com/jupyter/jupyter_client/issues/637 - 'jupyter-client>=6.1.11,<6.1.13', - 'timeloop>=1.0.2,<2', -] - -INTERACTIVE_BEAM_TEST = [ - # notebok utils - 'nbformat>=5.0.5,<6', - 'nbconvert>=6.2.0,<7', - # headless chrome based integration tests - 'needle>=0.5.0,<1', - 'chromedriver-binary>=100,<101', - # use a fixed major version of PIL for different python versions - 'pillow>=7.1.1,<8', -] - -AWS_REQUIREMENTS = ['boto3 >=1.9'] - -AZURE_REQUIREMENTS = [ - 'azure-storage-blob >=12.3.2', - 'azure-core >=1.7.0', -] - + pyarrow_dependency = '' +else: + pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0' # We must generate protos after setup_requires are installed. def generate_protos_first(): @@ -272,6 +162,8 @@ def get_portability_package_data(): # structure must exist before the call to setuptools.find_packages() # executes below. generate_protos_first() + # Keep all dependencies inlined in the setup call, otherwise Dependabot won't + # be able to parse it. setuptools.setup( name=PACKAGE_NAME, version=PACKAGE_VERSION, @@ -309,7 +201,39 @@ def get_portability_package_data(): 'apache_beam/utils/counters.py', 'apache_beam/utils/windowed_value.py', ]), - install_requires=REQUIRED_PACKAGES, + install_requires=[ + # Avro 1.9.2 for python3 was broken. + # The issue was fixed in version 1.9.2.1 + 'crcmod>=1.7,<2.0', + # dataclasses backport for python_version<3.7. No version bound because + # this is Python standard since Python 3.7 and each Python version is + # compatible with a specific dataclasses version. + 'dataclasses;python_version<"3.7"', + 'orjson<4.0', + # Dill doesn't have forwards-compatibility guarantees within minor + # version. Pickles created with a new version of dill may not unpickle + # using older version of dill. It is best to use the same version of + # dill on client and server, therefore list of allowed versions is very + # narrow. See: https://github.com/uqfoundation/dill/issues/341. + 'dill>=0.3.1.1,<0.3.2', + 'cloudpickle>=2.0.0,<3', + 'fastavro>=0.23.6,<2', + 'grpcio>=1.29.0,<2', + 'hdfs>=2.1.0,<3.0.0', + 'httplib2>=0.8,<0.21.0', + 'numpy>=1.14.3,<1.23.0', + 'pymongo>=3.8.0,<4.0.0', + 'protobuf>=3.12.2,<4', + 'proto-plus>=1.7.1,<2', + 'pydot>=1.2.0,<2', + 'python-dateutil>=2.8.0,<3', + 'pytz>=2018.3', + 'requests>=2.24.0,<3.0.0', + 'typing-extensions>=3.7.0', + # Dynamic dependencies must be specified in a separate list, otherwise + # Dependabot won't be able to parse the main list. Any dynamic + # dependencies will not receive updates from Dependabot. + ] + [pyarrow_dependency], python_requires=python_requires, # BEAM-8840: Do NOT use tests_require or setup_requires. extras_require={ @@ -319,12 +243,78 @@ def get_portability_package_data(): # https://github.com/sphinx-doc/sphinx/issues/9727 'docutils==0.17.1' ], - 'test': REQUIRED_TEST_PACKAGES, - 'gcp': GCP_REQUIREMENTS, - 'interactive': INTERACTIVE_BEAM, - 'interactive_test': INTERACTIVE_BEAM_TEST, - 'aws': AWS_REQUIREMENTS, - 'azure': AZURE_REQUIREMENTS, + 'test': [ + 'freezegun>=0.3.12', + 'joblib>=1.0.1', + 'mock>=1.0.1,<3.0.0', + 'pandas<2.0.0', + 'parameterized>=0.7.1,<0.8.0', + 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', + 'pyyaml>=3.12,<7.0.0', + 'requests_mock>=1.7,<2.0', + 'tenacity>=5.0.2,<6.0', + 'pytest>=4.4.0,<5.0', + 'pytest-xdist>=1.29.0,<2', + 'pytest-timeout>=1.3.3,<2', + 'scikit-learn>=0.20.0', + 'sqlalchemy>=1.3,<2.0', + 'psycopg2-binary>=2.8.5,<3.0.0', + 'testcontainers[mysql]>=3.0.3,<4.0.0', + 'cryptography>=36.0.0', + ], + 'gcp': [ + 'cachetools>=3.1.0,<5', + 'google-apitools>=0.5.31,<0.5.32', + # NOTE: Maintainers, please do not require google-auth>=2.x.x + # Until this issue is closed + # https://github.com/googleapis/google-cloud-python/issues/10566 + 'google-auth>=1.18.0,<3', + 'google-auth-httplib2>=0.1.0,<0.2.0', + 'google-cloud-datastore>=1.8.0,<2', + 'google-cloud-pubsub>=2.1.0,<3', + 'google-cloud-pubsublite>=1.2.0,<2', + # GCP packages required by tests + 'google-cloud-bigquery>=1.6.0,<3', + 'google-cloud-bigquery-storage>=2.6.3', + 'google-cloud-core>=0.28.1,<2', + 'google-cloud-bigtable>=0.31.1,<2', + 'google-cloud-spanner>=1.13.0,<2', + 'grpcio-gcp>=0.2.2,<1', + # GCP Packages required by ML functionality + 'google-cloud-dlp>=3.0.0,<4', + 'google-cloud-language>=1.3.0,<2', + 'google-cloud-videointelligence>=1.8.0,<2', + 'google-cloud-vision>=0.38.0,<2', + 'google-cloud-recommendations-ai>=0.1.0,<=0.2.0' + ], + 'interactive': [ + 'facets-overview>=1.0.0,<2', + 'google-cloud-dataproc>=3.0.0,<3.2.0', + # IPython>=8 is not compatible with Python<=3.7 + 'ipython>=7,<8;python_version<="3.7"', + 'ipython>=8,<9;python_version>"3.7"', + 'ipykernel>=6,<7', + 'ipywidgets>=7.6.5,<8', + # Skip version 6.1.13 due to + # https://github.com/jupyter/jupyter_client/issues/637 + 'jupyter-client>=6.1.11,<6.1.13', + 'timeloop>=1.0.2,<2', + ], + 'interactive_test': [ + # notebok utils + 'nbformat>=5.0.5,<6', + 'nbconvert>=6.2.0,<7', + # headless chrome based integration tests + 'needle>=0.5.0,<1', + 'chromedriver-binary>=100,<101', + # use a fixed major version of PIL for different python versions + 'pillow>=7.1.1,<8', + ], + 'aws': ['boto3 >=1.9'], + 'azure': [ + 'azure-storage-blob >=12.3.2', + 'azure-core >=1.7.0', + ], 'dataframe': ['pandas>=1.0,<1.5'] }, zip_safe=False,