From 5cd5fcb9b84eeb2aa4ff968df86d63f2a97b2b5c Mon Sep 17 00:00:00 2001 From: Minbo Bae Date: Thu, 5 Sep 2024 23:33:34 -0700 Subject: [PATCH 1/3] Add keyrings.google-artifactregistry-auth --- CHANGES.md | 1 + sdks/python/container/base_image_requirements_manual.txt | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c194cf730785..b5b571f13b91 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -66,6 +66,7 @@ ## New Features / Improvements +* Python worker can install packages from Google Artifact Registry Python repositories (Python) ([#32123](https://github.com/apache/beam/issues/32123)). * X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). ## Breaking Changes diff --git a/sdks/python/container/base_image_requirements_manual.txt b/sdks/python/container/base_image_requirements_manual.txt index 2d99a55e564e..586276310a72 100644 --- a/sdks/python/container/base_image_requirements_manual.txt +++ b/sdks/python/container/base_image_requirements_manual.txt @@ -38,4 +38,7 @@ nose==1.3.7 # For Dataflow internal testing. TODO: remove this. python-snappy # Optimizes execution of some Beam codepaths. scipy scikit-learn -build>=1.0,<2 # tool to build sdist from setup.py in stager. \ No newline at end of file +build>=1.0,<2 # tool to build sdist from setup.py in stager. +# Authentication for Google Artifact Registry when using --extra-index-url or --index-url in requirements.txt in +# Dataflow, which allows installing python packages from private Python repositories in GAR. +keyrings.google-artifactregistry-auth From 513df1c98236166672a797df77d9fbbc4261499e Mon Sep 17 00:00:00 2001 From: Minbo Bae Date: Fri, 6 Sep 2024 01:08:55 -0700 Subject: [PATCH 2/3] Update base image requirements. --- .../py310/base_image_requirements.txt | 51 +++++++++++-------- .../py311/base_image_requirements.txt | 51 +++++++++++-------- .../py312/base_image_requirements.txt | 50 ++++++++++-------- .../py38/base_image_requirements.txt | 49 ++++++++++-------- .../py39/base_image_requirements.txt | 51 +++++++++++-------- 5 files changed, 150 insertions(+), 102 deletions(-) diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index 017028abe8b7..c89f35e243f1 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -24,18 +24,19 @@ annotated-types==0.7.0 async-timeout==4.0.3 attrs==24.2.0 +backports.tarfile==1.2.0 beautifulsoup4==4.12.3 bs4==0.0.2 -build==1.2.1 +build==1.2.2 cachetools==5.5.0 -certifi==2024.7.4 -cffi==1.17.0 +certifi==2024.8.30 +cffi==1.17.1 charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 cramjam==2.8.3 crcmod==1.7 -cryptography==43.0.0 +cryptography==43.0.1 Cython==3.0.11 deprecation==2.1.0 dill==0.3.1.1 @@ -45,18 +46,18 @@ docopt==0.6.2 docstring_parser==0.16 exceptiongroup==1.2.2 execnet==2.1.1 -fastavro==1.9.5 +fastavro==1.9.7 fasteners==0.19 freezegun==1.5.1 future==1.0.0 -google-api-core==2.19.1 -google-api-python-client==2.142.0 +google-api-core==2.19.2 +google-api-python-client==2.144.0 google-apitools==0.5.31 google-auth==2.34.0 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.63.0 +google-cloud-aiplatform==1.65.0 google-cloud-bigquery==3.25.0 -google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery-storage==2.26.0 google-cloud-bigtable==2.26.0 google-cloud-core==2.4.1 google-cloud-datastore==2.20.1 @@ -67,33 +68,41 @@ google-cloud-pubsub==2.23.0 google-cloud-pubsublite==1.11.1 google-cloud-recommendations-ai==0.10.12 google-cloud-resource-manager==1.12.5 -google-cloud-spanner==3.48.0 +google-cloud-spanner==3.49.1 google-cloud-storage==2.18.2 google-cloud-videointelligence==2.13.5 google-cloud-vision==3.7.4 -google-crc32c==1.5.0 +google-crc32c==1.6.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.63.2 +googleapis-common-protos==1.65.0 greenlet==3.0.3 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 -grpcio==1.66.0 +grpcio==1.66.1 grpcio-status==1.62.3 guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.111.2 +hypothesis==6.112.0 idna==3.8 +importlib_metadata==8.4.0 iniconfig==2.0.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.0.2 +jeepney==0.8.0 Jinja2==3.1.4 joblib==1.4.2 Js2Py==0.74 -jsonpickle==3.2.2 +jsonpickle==3.3.0 jsonschema==4.23.0 jsonschema-specifications==2023.12.1 +keyring==25.3.0 +keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==2.1.5 mmh3==4.1.0 mock==5.1.0 +more-itertools==10.5.0 nltk==3.9.1 nose==1.3.7 numpy==1.26.4 @@ -113,20 +122,20 @@ pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 pycparser==2.22 -pydantic==2.8.2 -pydantic_core==2.20.1 +pydantic==2.9.0 +pydantic_core==2.23.2 pydot==1.4.2 PyHamcrest==2.1.0 pyjsparser==2.7.1 pymongo==4.8.0 PyMySQL==1.1.1 -pyparsing==3.1.2 +pyparsing==3.1.4 pyproject_hooks==1.1.0 pytest==7.4.4 pytest-timeout==2.3.1 pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 -python-snappy==0.7.2 +python-snappy==0.7.3 pytz==2024.1 PyYAML==6.0.2 redis==5.0.8 @@ -138,11 +147,12 @@ rpds-py==0.20.0 rsa==4.9 scikit-learn==1.5.1 scipy==1.14.1 +SecretStorage==3.3.3 shapely==2.0.6 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.6 -SQLAlchemy==2.0.32 +SQLAlchemy==2.0.34 sqlparse==0.5.1 tenacity==8.5.0 testcontainers==3.7.1 @@ -155,4 +165,5 @@ tzlocal==5.2 uritemplate==4.1.1 urllib3==2.2.2 wrapt==1.16.0 +zipp==3.20.1 zstandard==0.23.0 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index 622cbad35c7e..fda2e2146184 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -23,18 +23,19 @@ annotated-types==0.7.0 attrs==24.2.0 +backports.tarfile==1.2.0 beautifulsoup4==4.12.3 bs4==0.0.2 -build==1.2.1 +build==1.2.2 cachetools==5.5.0 -certifi==2024.7.4 -cffi==1.17.0 +certifi==2024.8.30 +cffi==1.17.1 charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 cramjam==2.8.3 crcmod==1.7 -cryptography==43.0.0 +cryptography==43.0.1 Cython==3.0.11 deprecation==2.1.0 dill==0.3.1.1 @@ -43,18 +44,18 @@ docker==7.1.0 docopt==0.6.2 docstring_parser==0.16 execnet==2.1.1 -fastavro==1.9.5 +fastavro==1.9.7 fasteners==0.19 freezegun==1.5.1 future==1.0.0 -google-api-core==2.19.1 -google-api-python-client==2.142.0 +google-api-core==2.19.2 +google-api-python-client==2.144.0 google-apitools==0.5.31 google-auth==2.34.0 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.63.0 +google-cloud-aiplatform==1.65.0 google-cloud-bigquery==3.25.0 -google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery-storage==2.26.0 google-cloud-bigtable==2.26.0 google-cloud-core==2.4.1 google-cloud-datastore==2.20.1 @@ -65,33 +66,41 @@ google-cloud-pubsub==2.23.0 google-cloud-pubsublite==1.11.1 google-cloud-recommendations-ai==0.10.12 google-cloud-resource-manager==1.12.5 -google-cloud-spanner==3.48.0 +google-cloud-spanner==3.49.1 google-cloud-storage==2.18.2 google-cloud-videointelligence==2.13.5 google-cloud-vision==3.7.4 -google-crc32c==1.5.0 +google-crc32c==1.6.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.63.2 +googleapis-common-protos==1.65.0 greenlet==3.0.3 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 -grpcio==1.66.0 +grpcio==1.66.1 grpcio-status==1.62.3 guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.111.2 +hypothesis==6.112.0 idna==3.8 +importlib_metadata==8.4.0 iniconfig==2.0.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.0.2 +jeepney==0.8.0 Jinja2==3.1.4 joblib==1.4.2 Js2Py==0.74 -jsonpickle==3.2.2 +jsonpickle==3.3.0 jsonschema==4.23.0 jsonschema-specifications==2023.12.1 +keyring==25.3.0 +keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==2.1.5 mmh3==4.1.0 mock==5.1.0 +more-itertools==10.5.0 nltk==3.9.1 nose==1.3.7 numpy==1.26.4 @@ -111,20 +120,20 @@ pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 pycparser==2.22 -pydantic==2.8.2 -pydantic_core==2.20.1 +pydantic==2.9.0 +pydantic_core==2.23.2 pydot==1.4.2 PyHamcrest==2.1.0 pyjsparser==2.7.1 pymongo==4.8.0 PyMySQL==1.1.1 -pyparsing==3.1.2 +pyparsing==3.1.4 pyproject_hooks==1.1.0 pytest==7.4.4 pytest-timeout==2.3.1 pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 -python-snappy==0.7.2 +python-snappy==0.7.3 pytz==2024.1 PyYAML==6.0.2 redis==5.0.8 @@ -136,11 +145,12 @@ rpds-py==0.20.0 rsa==4.9 scikit-learn==1.5.1 scipy==1.14.1 +SecretStorage==3.3.3 shapely==2.0.6 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.6 -SQLAlchemy==2.0.32 +SQLAlchemy==2.0.34 sqlparse==0.5.1 tenacity==8.5.0 testcontainers==3.7.1 @@ -152,4 +162,5 @@ tzlocal==5.2 uritemplate==4.1.1 urllib3==2.2.2 wrapt==1.16.0 +zipp==3.20.1 zstandard==0.23.0 diff --git a/sdks/python/container/py312/base_image_requirements.txt b/sdks/python/container/py312/base_image_requirements.txt index 0bd94e2da124..c027191bc6aa 100644 --- a/sdks/python/container/py312/base_image_requirements.txt +++ b/sdks/python/container/py312/base_image_requirements.txt @@ -25,16 +25,16 @@ annotated-types==0.7.0 attrs==24.2.0 beautifulsoup4==4.12.3 bs4==0.0.2 -build==1.2.1 +build==1.2.2 cachetools==5.5.0 -certifi==2024.7.4 -cffi==1.17.0 +certifi==2024.8.30 +cffi==1.17.1 charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 cramjam==2.8.3 crcmod==1.7 -cryptography==43.0.0 +cryptography==43.0.1 Cython==3.0.11 deprecation==2.1.0 dill==0.3.1.1 @@ -43,18 +43,18 @@ docker==7.1.0 docopt==0.6.2 docstring_parser==0.16 execnet==2.1.1 -fastavro==1.9.5 +fastavro==1.9.7 fasteners==0.19 freezegun==1.5.1 future==1.0.0 -google-api-core==2.19.1 -google-api-python-client==2.142.0 +google-api-core==2.19.2 +google-api-python-client==2.144.0 google-apitools==0.5.31 google-auth==2.34.0 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.63.0 +google-cloud-aiplatform==1.65.0 google-cloud-bigquery==3.25.0 -google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery-storage==2.26.0 google-cloud-bigtable==2.26.0 google-cloud-core==2.4.1 google-cloud-datastore==2.20.1 @@ -65,32 +65,39 @@ google-cloud-pubsub==2.23.0 google-cloud-pubsublite==1.11.1 google-cloud-recommendations-ai==0.10.12 google-cloud-resource-manager==1.12.5 -google-cloud-spanner==3.48.0 +google-cloud-spanner==3.49.1 google-cloud-storage==2.18.2 google-cloud-videointelligence==2.13.5 google-cloud-vision==3.7.4 -google-crc32c==1.5.0 +google-crc32c==1.6.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.63.2 +googleapis-common-protos==1.65.0 greenlet==3.0.3 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 -grpcio==1.66.0 +grpcio==1.66.1 grpcio-status==1.62.3 guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.111.2 +hypothesis==6.112.0 idna==3.8 iniconfig==2.0.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.0.2 +jeepney==0.8.0 Jinja2==3.1.4 joblib==1.4.2 -jsonpickle==3.2.2 +jsonpickle==3.3.0 jsonschema==4.23.0 jsonschema-specifications==2023.12.1 +keyring==25.3.0 +keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==2.1.5 mmh3==4.1.0 mock==5.1.0 +more-itertools==10.5.0 nltk==3.9.1 nose==1.3.7 numpy==1.26.4 @@ -110,19 +117,19 @@ pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 pycparser==2.22 -pydantic==2.8.2 -pydantic_core==2.20.1 +pydantic==2.9.0 +pydantic_core==2.23.2 pydot==1.4.2 PyHamcrest==2.1.0 pymongo==4.8.0 PyMySQL==1.1.1 -pyparsing==3.1.2 +pyparsing==3.1.4 pyproject_hooks==1.1.0 pytest==7.4.4 pytest-timeout==2.3.1 pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 -python-snappy==0.7.2 +python-snappy==0.7.3 pytz==2024.1 PyYAML==6.0.2 redis==5.0.8 @@ -134,12 +141,13 @@ rpds-py==0.20.0 rsa==4.9 scikit-learn==1.5.1 scipy==1.14.1 -setuptools==73.0.1 +SecretStorage==3.3.3 +setuptools==74.1.2 shapely==2.0.6 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.6 -SQLAlchemy==2.0.32 +SQLAlchemy==2.0.34 sqlparse==0.5.1 tenacity==8.5.0 testcontainers==3.7.1 diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt index 699057ee5156..1796e4e83bfa 100644 --- a/sdks/python/container/py38/base_image_requirements.txt +++ b/sdks/python/container/py38/base_image_requirements.txt @@ -24,19 +24,20 @@ annotated-types==0.7.0 async-timeout==4.0.3 attrs==24.2.0 +backports.tarfile==1.2.0 backports.zoneinfo==0.2.1 beautifulsoup4==4.12.3 bs4==0.0.2 -build==1.2.1 +build==1.2.2 cachetools==5.5.0 -certifi==2024.7.4 -cffi==1.17.0 +certifi==2024.8.30 +cffi==1.17.1 charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 cramjam==2.8.3 crcmod==1.7 -cryptography==43.0.0 +cryptography==43.0.1 Cython==3.0.11 deprecation==2.1.0 dill==0.3.1.1 @@ -46,18 +47,18 @@ docopt==0.6.2 docstring_parser==0.16 exceptiongroup==1.2.2 execnet==2.1.1 -fastavro==1.9.5 +fastavro==1.9.7 fasteners==0.19 freezegun==1.5.1 future==1.0.0 -google-api-core==2.19.1 -google-api-python-client==2.142.0 +google-api-core==2.19.2 +google-api-python-client==2.144.0 google-apitools==0.5.31 google-auth==2.34.0 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.63.0 +google-cloud-aiplatform==1.65.0 google-cloud-bigquery==3.25.0 -google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery-storage==2.26.0 google-cloud-bigtable==2.26.0 google-cloud-core==2.4.1 google-cloud-datastore==2.20.1 @@ -68,35 +69,42 @@ google-cloud-pubsub==2.23.0 google-cloud-pubsublite==1.11.1 google-cloud-recommendations-ai==0.10.12 google-cloud-resource-manager==1.12.5 -google-cloud-spanner==3.48.0 +google-cloud-spanner==3.49.1 google-cloud-storage==2.18.2 google-cloud-videointelligence==2.13.5 google-cloud-vision==3.7.4 google-crc32c==1.5.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.63.2 +googleapis-common-protos==1.65.0 greenlet==3.0.3 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 -grpcio==1.66.0 +grpcio==1.66.1 grpcio-status==1.62.3 guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.111.2 +hypothesis==6.112.0 idna==3.8 importlib_metadata==8.4.0 importlib_resources==6.4.4 iniconfig==2.0.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.0.2 +jeepney==0.8.0 Jinja2==3.1.4 joblib==1.4.2 Js2Py==0.74 -jsonpickle==3.2.2 +jsonpickle==3.3.0 jsonschema==4.23.0 jsonschema-specifications==2023.12.1 +keyring==25.3.0 +keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==2.1.5 mmh3==4.1.0 mock==5.1.0 +more-itertools==10.5.0 nltk==3.9.1 nose==1.3.7 numpy==1.24.4 @@ -117,20 +125,20 @@ pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 pycparser==2.22 -pydantic==2.8.2 -pydantic_core==2.20.1 +pydantic==2.9.0 +pydantic_core==2.23.2 pydot==1.4.2 PyHamcrest==2.1.0 pyjsparser==2.7.1 pymongo==4.8.0 PyMySQL==1.1.1 -pyparsing==3.1.2 +pyparsing==3.1.4 pyproject_hooks==1.1.0 pytest==7.4.4 pytest-timeout==2.3.1 pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 -python-snappy==0.7.2 +python-snappy==0.7.3 pytz==2024.1 PyYAML==6.0.2 redis==5.0.8 @@ -142,11 +150,12 @@ rpds-py==0.20.0 rsa==4.9 scikit-learn==1.3.2 scipy==1.10.1 +SecretStorage==3.3.3 shapely==2.0.6 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.6 -SQLAlchemy==2.0.32 +SQLAlchemy==2.0.34 sqlparse==0.5.1 tenacity==8.5.0 testcontainers==3.7.1 @@ -159,5 +168,5 @@ tzlocal==5.2 uritemplate==4.1.1 urllib3==2.2.2 wrapt==1.16.0 -zipp==3.20.0 +zipp==3.20.1 zstandard==0.23.0 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index d8546490c182..aa2b4f3bb388 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -24,18 +24,19 @@ annotated-types==0.7.0 async-timeout==4.0.3 attrs==24.2.0 +backports.tarfile==1.2.0 beautifulsoup4==4.12.3 bs4==0.0.2 -build==1.2.1 +build==1.2.2 cachetools==5.5.0 -certifi==2024.7.4 -cffi==1.17.0 +certifi==2024.8.30 +cffi==1.17.1 charset-normalizer==3.3.2 click==8.1.7 cloudpickle==2.2.1 cramjam==2.8.3 crcmod==1.7 -cryptography==43.0.0 +cryptography==43.0.1 Cython==3.0.11 deprecation==2.1.0 dill==0.3.1.1 @@ -45,18 +46,18 @@ docopt==0.6.2 docstring_parser==0.16 exceptiongroup==1.2.2 execnet==2.1.1 -fastavro==1.9.5 +fastavro==1.9.7 fasteners==0.19 freezegun==1.5.1 future==1.0.0 -google-api-core==2.19.1 -google-api-python-client==2.142.0 +google-api-core==2.19.2 +google-api-python-client==2.144.0 google-apitools==0.5.31 google-auth==2.34.0 google-auth-httplib2==0.2.0 -google-cloud-aiplatform==1.63.0 +google-cloud-aiplatform==1.65.0 google-cloud-bigquery==3.25.0 -google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery-storage==2.26.0 google-cloud-bigtable==2.26.0 google-cloud-core==2.4.1 google-cloud-datastore==2.20.1 @@ -67,34 +68,41 @@ google-cloud-pubsub==2.23.0 google-cloud-pubsublite==1.11.1 google-cloud-recommendations-ai==0.10.12 google-cloud-resource-manager==1.12.5 -google-cloud-spanner==3.48.0 +google-cloud-spanner==3.49.1 google-cloud-storage==2.18.2 google-cloud-videointelligence==2.13.5 google-cloud-vision==3.7.4 -google-crc32c==1.5.0 +google-crc32c==1.6.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.63.2 +googleapis-common-protos==1.65.0 greenlet==3.0.3 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 -grpcio==1.66.0 +grpcio==1.66.1 grpcio-status==1.62.3 guppy3==3.1.4.post1 hdfs==2.7.3 httplib2==0.22.0 -hypothesis==6.111.2 +hypothesis==6.112.0 idna==3.8 importlib_metadata==8.4.0 iniconfig==2.0.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.0.2 +jeepney==0.8.0 Jinja2==3.1.4 joblib==1.4.2 Js2Py==0.74 -jsonpickle==3.2.2 +jsonpickle==3.3.0 jsonschema==4.23.0 jsonschema-specifications==2023.12.1 +keyring==25.3.0 +keyrings.google-artifactregistry-auth==1.1.2 MarkupSafe==2.1.5 mmh3==4.1.0 mock==5.1.0 +more-itertools==10.5.0 nltk==3.9.1 nose==1.3.7 numpy==1.26.4 @@ -114,20 +122,20 @@ pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 pycparser==2.22 -pydantic==2.8.2 -pydantic_core==2.20.1 +pydantic==2.9.0 +pydantic_core==2.23.2 pydot==1.4.2 PyHamcrest==2.1.0 pyjsparser==2.7.1 pymongo==4.8.0 PyMySQL==1.1.1 -pyparsing==3.1.2 +pyparsing==3.1.4 pyproject_hooks==1.1.0 pytest==7.4.4 pytest-timeout==2.3.1 pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 -python-snappy==0.7.2 +python-snappy==0.7.3 pytz==2024.1 PyYAML==6.0.2 redis==5.0.8 @@ -139,11 +147,12 @@ rpds-py==0.20.0 rsa==4.9 scikit-learn==1.5.1 scipy==1.13.1 +SecretStorage==3.3.3 shapely==2.0.6 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.6 -SQLAlchemy==2.0.32 +SQLAlchemy==2.0.34 sqlparse==0.5.1 tenacity==8.5.0 testcontainers==3.7.1 @@ -156,5 +165,5 @@ tzlocal==5.2 uritemplate==4.1.1 urllib3==2.2.2 wrapt==1.16.0 -zipp==3.20.0 +zipp==3.20.1 zstandard==0.23.0 From bab40ffee57172e159b419521226a254db75b03e Mon Sep 17 00:00:00 2001 From: Minbo Bae Date: Sat, 7 Sep 2024 01:30:29 +0000 Subject: [PATCH 3/3] Move keyrings.google-artifactregistry-auth to [gcp] extension in setup.py --- CHANGES.md | 2 +- sdks/python/container/base_image_requirements_manual.txt | 3 --- sdks/python/setup.py | 7 ++++++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b5b571f13b91..9f9b2f6f80f6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -66,7 +66,7 @@ ## New Features / Improvements -* Python worker can install packages from Google Artifact Registry Python repositories (Python) ([#32123](https://github.com/apache/beam/issues/32123)). +* Dataflow worker can install packages from Google Artifact Registry Python repositories (Python) ([#32123](https://github.com/apache/beam/issues/32123)). * X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)). ## Breaking Changes diff --git a/sdks/python/container/base_image_requirements_manual.txt b/sdks/python/container/base_image_requirements_manual.txt index 586276310a72..570800b45f96 100644 --- a/sdks/python/container/base_image_requirements_manual.txt +++ b/sdks/python/container/base_image_requirements_manual.txt @@ -39,6 +39,3 @@ python-snappy # Optimizes execution of some Beam codepaths. scipy scikit-learn build>=1.0,<2 # tool to build sdist from setup.py in stager. -# Authentication for Google Artifact Registry when using --extra-index-url or --index-url in requirements.txt in -# Dataflow, which allows installing python packages from private Python repositories in GAR. -keyrings.google-artifactregistry-auth diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 3cd4e0693269..f50a3a07746f 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -450,7 +450,12 @@ def get_portability_package_data(): 'google-cloud-videointelligence>=2.0,<3', 'google-cloud-vision>=2,<4', 'google-cloud-recommendations-ai>=0.1.0,<0.11.0', - 'google-cloud-aiplatform>=1.26.0, < 2.0' + 'google-cloud-aiplatform>=1.26.0, < 2.0', + # Authentication for Google Artifact Registry when using + # --extra-index-url or --index-url in requirements.txt in + # Dataflow, which allows installing python packages from private + # Python repositories in GAR. + 'keyrings.google-artifactregistry-auth' ], 'interactive': [ 'facets-overview>=1.1.0,<2',