Skip to content

Commit

Permalink
[BEAM-12526] Add Dependabot (apache#17563)
Browse files Browse the repository at this point in the history 
* Create dependabot.yml

* Apache header

* See if dependabot will parse now

* Explanatory comment

* Reduce duplication

* Bug + style from feedback

* Change format to make dependabot happy
  • Loading branch information
@damccorm
damccorm authored May 11, 2022
1 parent f30e4e5 commit fbc495c
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 120 deletions.
29 changes: 29 additions & 0 deletions .github/dependabot.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version: 2
updates:
- package-ecosystem: "gomod"
directory: "/sdks" # Location of package manifests
schedule:
interval: "daily"
- package-ecosystem: "pip"
directory: "/sdks/python" # Location of package manifests
schedule:
interval: "daily"
- package-ecosystem: "gradle"
directory: "/" # Location of package manifests
schedule:
interval: "daily"
230 changes: 110 additions & 120 deletions sdks/python/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,121 +120,11 @@ def get_version():
except ImportError:
cythonize = lambda *args, **kwargs: []

REQUIRED_PACKAGES = [
# Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1
'crcmod>=1.7,<2.0',
# dataclasses backport for python_version<3.7. No version bound because this
# is Python standard since Python 3.7 and each Python version is compatible
# with a specific dataclasses version.
'dataclasses;python_version<"3.7"',
'orjson<4.0',
# Dill doesn't have forwards-compatibility guarantees within minor version.
# Pickles created with a new version of dill may not unpickle using older
# version of dill. It is best to use the same version of dill on client and
# server, therefore list of allowed versions is very narrow.
# See: https://github.com/uqfoundation/dill/issues/341.
'dill>=0.3.1.1,<0.3.2',
'cloudpickle>=2.0.0,<3',
'fastavro>=0.23.6,<2',
'grpcio>=1.29.0,<2',
'hdfs>=2.1.0,<3.0.0',
'httplib2>=0.8,<0.21.0',
'numpy>=1.14.3,<1.23.0',
'pymongo>=3.8.0,<4.0.0',
'protobuf>=3.12.2,<4',
'proto-plus>=1.7.1,<2',
'pyarrow>=0.15.1,<8.0.0',
'pydot>=1.2.0,<2',
'python-dateutil>=2.8.0,<3',
'pytz>=2018.3',
'requests>=2.24.0,<3.0.0',
'typing-extensions>=3.7.0',
]

# [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.
if sys.platform == 'win32' and sys.maxsize <= 2**32:
REQUIRED_PACKAGES = [
p for p in REQUIRED_PACKAGES if not p.startswith('pyarrow')
]

REQUIRED_TEST_PACKAGES = [
'freezegun>=0.3.12',
'joblib>=1.0.1',
'mock>=1.0.1,<3.0.0',
'pandas<2.0.0',
'parameterized>=0.7.1,<0.8.0',
'pyhamcrest>=1.9,!=1.10.0,<2.0.0',
'pyyaml>=3.12,<7.0.0',
'requests_mock>=1.7,<2.0',
'tenacity>=5.0.2,<6.0',
'pytest>=4.4.0,<5.0',
'pytest-xdist>=1.29.0,<2',
'pytest-timeout>=1.3.3,<2',
'scikit-learn>=0.20.0',
'sqlalchemy>=1.3,<2.0',
'psycopg2-binary>=2.8.5,<3.0.0',
'testcontainers[mysql]>=3.0.3,<4.0.0',
'cryptography>=36.0.0',
]

GCP_REQUIREMENTS = [
'cachetools>=3.1.0,<5',
'google-apitools>=0.5.31,<0.5.32',
# NOTE: Maintainers, please do not require google-auth>=2.x.x
# Until this issue is closed
# https://github.com/googleapis/google-cloud-python/issues/10566
'google-auth>=1.18.0,<3',
'google-auth-httplib2>=0.1.0,<0.2.0',
'google-cloud-datastore>=1.8.0,<2',
'google-cloud-pubsub>=2.1.0,<3',
'google-cloud-pubsublite>=1.2.0,<2',
# GCP packages required by tests
'google-cloud-bigquery>=1.6.0,<3',
'google-cloud-bigquery-storage>=2.6.3',
'google-cloud-core>=0.28.1,<2',
'google-cloud-bigtable>=0.31.1,<2',
'google-cloud-spanner>=1.13.0,<2',
'grpcio-gcp>=0.2.2,<1',
# GCP Packages required by ML functionality
'google-cloud-dlp>=3.0.0,<4',
'google-cloud-language>=1.3.0,<2',
'google-cloud-videointelligence>=1.8.0,<2',
'google-cloud-vision>=0.38.0,<2',
'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'
]

INTERACTIVE_BEAM = [
'facets-overview>=1.0.0,<2',
'google-cloud-dataproc>=3.0.0,<3.2.0',
# IPython>=8 is not compatible with Python<=3.7
'ipython>=7,<8;python_version<="3.7"',
'ipython>=8,<9;python_version>"3.7"',
'ipykernel>=6,<7',
'ipywidgets>=7.6.5,<8',
# Skip version 6.1.13 due to
# https://github.com/jupyter/jupyter_client/issues/637
'jupyter-client>=6.1.11,<6.1.13',
'timeloop>=1.0.2,<2',
]

INTERACTIVE_BEAM_TEST = [
# notebok utils
'nbformat>=5.0.5,<6',
'nbconvert>=6.2.0,<7',
# headless chrome based integration tests
'needle>=0.5.0,<1',
'chromedriver-binary>=100,<101',
# use a fixed major version of PIL for different python versions
'pillow>=7.1.1,<8',
]

AWS_REQUIREMENTS = ['boto3 >=1.9']

AZURE_REQUIREMENTS = [
'azure-storage-blob >=12.3.2',
'azure-core >=1.7.0',
]

pyarrow_dependency = ''
else:
pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0'

# We must generate protos after setup_requires are installed.
def generate_protos_first():
Expand Down Expand Up @@ -272,6 +162,8 @@ def get_portability_package_data():
# structure must exist before the call to setuptools.find_packages()
# executes below.
generate_protos_first()
# Keep all dependencies inlined in the setup call, otherwise Dependabot won't
# be able to parse it.
setuptools.setup(
name=PACKAGE_NAME,
version=PACKAGE_VERSION,
Expand Down Expand Up @@ -309,7 +201,39 @@ def get_portability_package_data():
'apache_beam/utils/counters.py',
'apache_beam/utils/windowed_value.py',
]),
install_requires=REQUIRED_PACKAGES,
install_requires=[
# Avro 1.9.2 for python3 was broken.
# The issue was fixed in version 1.9.2.1
'crcmod>=1.7,<2.0',
# dataclasses backport for python_version<3.7. No version bound because
# this is Python standard since Python 3.7 and each Python version is
# compatible with a specific dataclasses version.
'dataclasses;python_version<"3.7"',
'orjson<4.0',
# Dill doesn't have forwards-compatibility guarantees within minor
# version. Pickles created with a new version of dill may not unpickle
# using older version of dill. It is best to use the same version of
# dill on client and server, therefore list of allowed versions is very
# narrow. See: https://github.com/uqfoundation/dill/issues/341.
'dill>=0.3.1.1,<0.3.2',
'cloudpickle>=2.0.0,<3',
'fastavro>=0.23.6,<2',
'grpcio>=1.29.0,<2',
'hdfs>=2.1.0,<3.0.0',
'httplib2>=0.8,<0.21.0',
'numpy>=1.14.3,<1.23.0',
'pymongo>=3.8.0,<4.0.0',
'protobuf>=3.12.2,<4',
'proto-plus>=1.7.1,<2',
'pydot>=1.2.0,<2',
'python-dateutil>=2.8.0,<3',
'pytz>=2018.3',
'requests>=2.24.0,<3.0.0',
'typing-extensions>=3.7.0',
# Dynamic dependencies must be specified in a separate list, otherwise
# Dependabot won't be able to parse the main list. Any dynamic
# dependencies will not receive updates from Dependabot.
] + [pyarrow_dependency],
python_requires=python_requires,
# BEAM-8840: Do NOT use tests_require or setup_requires.
extras_require={
Expand All @@ -319,12 +243,78 @@ def get_portability_package_data():
# https://github.com/sphinx-doc/sphinx/issues/9727
'docutils==0.17.1'
],
'test': REQUIRED_TEST_PACKAGES,
'gcp': GCP_REQUIREMENTS,
'interactive': INTERACTIVE_BEAM,
'interactive_test': INTERACTIVE_BEAM_TEST,
'aws': AWS_REQUIREMENTS,
'azure': AZURE_REQUIREMENTS,
'test': [
'freezegun>=0.3.12',
'joblib>=1.0.1',
'mock>=1.0.1,<3.0.0',
'pandas<2.0.0',
'parameterized>=0.7.1,<0.8.0',
'pyhamcrest>=1.9,!=1.10.0,<2.0.0',
'pyyaml>=3.12,<7.0.0',
'requests_mock>=1.7,<2.0',
'tenacity>=5.0.2,<6.0',
'pytest>=4.4.0,<5.0',
'pytest-xdist>=1.29.0,<2',
'pytest-timeout>=1.3.3,<2',
'scikit-learn>=0.20.0',
'sqlalchemy>=1.3,<2.0',
'psycopg2-binary>=2.8.5,<3.0.0',
'testcontainers[mysql]>=3.0.3,<4.0.0',
'cryptography>=36.0.0',
],
'gcp': [
'cachetools>=3.1.0,<5',
'google-apitools>=0.5.31,<0.5.32',
# NOTE: Maintainers, please do not require google-auth>=2.x.x
# Until this issue is closed
# https://github.com/googleapis/google-cloud-python/issues/10566
'google-auth>=1.18.0,<3',
'google-auth-httplib2>=0.1.0,<0.2.0',
'google-cloud-datastore>=1.8.0,<2',
'google-cloud-pubsub>=2.1.0,<3',
'google-cloud-pubsublite>=1.2.0,<2',
# GCP packages required by tests
'google-cloud-bigquery>=1.6.0,<3',
'google-cloud-bigquery-storage>=2.6.3',
'google-cloud-core>=0.28.1,<2',
'google-cloud-bigtable>=0.31.1,<2',
'google-cloud-spanner>=1.13.0,<2',
'grpcio-gcp>=0.2.2,<1',
# GCP Packages required by ML functionality
'google-cloud-dlp>=3.0.0,<4',
'google-cloud-language>=1.3.0,<2',
'google-cloud-videointelligence>=1.8.0,<2',
'google-cloud-vision>=0.38.0,<2',
'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'
],
'interactive': [
'facets-overview>=1.0.0,<2',
'google-cloud-dataproc>=3.0.0,<3.2.0',
# IPython>=8 is not compatible with Python<=3.7
'ipython>=7,<8;python_version<="3.7"',
'ipython>=8,<9;python_version>"3.7"',
'ipykernel>=6,<7',
'ipywidgets>=7.6.5,<8',
# Skip version 6.1.13 due to
# https://github.com/jupyter/jupyter_client/issues/637
'jupyter-client>=6.1.11,<6.1.13',
'timeloop>=1.0.2,<2',
],
'interactive_test': [
# notebok utils
'nbformat>=5.0.5,<6',
'nbconvert>=6.2.0,<7',
# headless chrome based integration tests
'needle>=0.5.0,<1',
'chromedriver-binary>=100,<101',
# use a fixed major version of PIL for different python versions
'pillow>=7.1.1,<8',
],
'aws': ['boto3 >=1.9'],
'azure': [
'azure-storage-blob >=12.3.2',
'azure-core >=1.7.0',
],
'dataframe': ['pandas>=1.0,<1.5']
},
zip_safe=False,
Expand Down

0 comments on commit fbc495c

Please sign in to comment.