backend: Pin python dependencies

Builds/tests failed due to broken transident dependencies. This pins the python dependencies to prevent this in the future.
kubeflow · Feb 26, 2020 · 08aea77 · 08aea77
1 parent f71c69c
commit 08aea77
Show file tree

Hide file tree

Showing 8 changed files with 310 additions and 17 deletions.
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -27,8 +27,8 @@ RUN if [ "$use_remote_build" = "true" ]; then \
 FROM python:3.5 as compiler
 RUN apt-get update -y && \
     apt-get install --no-install-recommends -y -q default-jdk python3-setuptools python3-dev
-RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
-RUN python3 -m pip install tfx==0.21.0
+COPY backend/requirements.txt .
+RUN python3 -m pip install -r requirements.txt
 
 WORKDIR /go/src/github.com/kubeflow/pipelines
 COPY sdk sdk

diff --git a/backend/README.md b/backend/README.md
@@ -43,3 +43,11 @@ If a new external Go dependency is added, or an existing one has its version
 bumped in the `go.mod` file, ensure the BUILD files pick this up by updating the
 WORKSPACE go_repository rules using the following command: `bazel run
 //:gazelle -- update-repos --from_file=go.mod`
+
+## Updating python dependencies
+
+[pip-tools](https://github.com/jazzband/pip-tools) is used to manage python
+dependencies. To update dependencies, edit [requirements.in](requirements.in)
+and run `./update_requirements.sh` to update and pin the transitive
+dependencies.
+
diff --git a/backend/requirements.in b/backend/requirements.in
@@ -0,0 +1 @@
+tfx==0.21.0
diff --git a/backend/requirements.txt b/backend/requirements.txt
@@ -0,0 +1,134 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile requirements.in
+#
+absl-py==0.8.1            # via ml-metadata, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl
+apache-beam[gcp]==2.17.0  # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl
+astor==0.8.1              # via tensorflow
+attrs==19.3.0             # via jsonschema
+avro-python3==1.9.1       # via apache-beam, tensorflow-data-validation, tensorflow-model-analysis, tfx, tfx-bsl
+backcall==0.1.0           # via ipython
+bleach==3.1.1             # via nbconvert
+cachetools==3.1.1         # via apache-beam, google-auth
+certifi==2019.11.28       # via requests
+chardet==3.0.4            # via requests
+click==7.0                # via tfx
+crcmod==1.7               # via apache-beam
+decorator==4.4.1          # via ipython, traitlets
+defusedxml==0.6.0         # via nbconvert
+dill==0.3.0               # via apache-beam
+docker==4.2.0             # via tfx
+docopt==0.6.2             # via hdfs
+entrypoints==0.3          # via nbconvert
+fastavro==0.21.24         # via apache-beam
+fasteners==0.15           # via google-apitools
+future==0.18.2            # via apache-beam
+gast==0.2.2               # via tensorflow
+google-api-core[grpc]==1.16.0  # via google-cloud-bigtable, google-cloud-core, google-cloud-datastore, google-cloud-pubsub
+google-api-python-client==1.7.11  # via tfx
+google-apitools==0.5.28   # via apache-beam
+google-auth-httplib2==0.0.3  # via google-api-python-client
+google-auth-oauthlib==0.4.1  # via tensorboard
+google-auth==1.11.2       # via google-api-core, google-api-python-client, google-auth-httplib2, google-auth-oauthlib, tensorboard
+google-cloud-bigquery==1.17.1  # via apache-beam
+google-cloud-bigtable==1.0.0  # via apache-beam
+google-cloud-core==1.3.0  # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore
+google-cloud-datastore==1.7.4  # via apache-beam
+google-cloud-pubsub==1.0.2  # via apache-beam
+google-pasta==0.1.8       # via tensorflow
+google-resumable-media==0.4.1  # via google-cloud-bigquery
+googleapis-common-protos[grpc]==1.51.0  # via google-api-core, grpc-google-iam-v1, tensorflow-metadata
+grpc-google-iam-v1==0.12.3  # via google-cloud-bigtable, google-cloud-pubsub
+grpcio==1.27.2            # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, tensorboard, tensorflow, tensorflow-serving-api, tfx
+h5py==2.10.0              # via keras-applications
+hdfs==2.5.8               # via apache-beam
+httplib2==0.12.0          # via apache-beam, google-api-python-client, google-apitools, google-auth-httplib2, oauth2client
+idna==2.9                 # via requests
+importlib-metadata==1.5.0  # via jsonschema
+ipykernel==5.1.4          # via ipywidgets, jupyter, jupyter-console, notebook, qtconsole
+ipython-genutils==0.2.0   # via nbformat, notebook, qtconsole, traitlets
+ipython==7.9.0            # via ipykernel, ipywidgets, jupyter-console, tensorflow-data-validation
+ipywidgets==7.5.1         # via jupyter, tensorflow-model-analysis
+jedi==0.16.0              # via ipython
+jinja2==2.11.1            # via nbconvert, notebook, tfx
+joblib==0.14.1            # via scikit-learn, tensorflow-data-validation
+jsonschema==3.2.0         # via nbformat
+jupyter-client==6.0.0     # via ipykernel, jupyter-console, notebook, qtconsole
+jupyter-console==6.1.0    # via jupyter
+jupyter-core==4.6.3       # via jupyter-client, nbconvert, nbformat, notebook, qtconsole
+jupyter==1.0.0            # via tensorflow-model-analysis
+keras-applications==1.0.8  # via tensorflow
+keras-preprocessing==1.1.0  # via tensorflow
+markdown==3.2.1           # via tensorboard
+markupsafe==1.1.1         # via jinja2
+mistune==0.8.4            # via nbconvert
+ml-metadata==0.21.1       # via tfx
+mock==2.0.0               # via apache-beam
+monotonic==1.5            # via fasteners
+nbconvert==5.6.1          # via jupyter, notebook
+nbformat==5.0.4           # via ipywidgets, nbconvert, notebook
+notebook==6.0.3           # via jupyter, widgetsnbextension
+numpy==1.18.1             # via h5py, keras-applications, keras-preprocessing, opt-einsum, pandas, pyarrow, scikit-learn, scipy, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx-bsl
+oauth2client==3.0.0       # via apache-beam, google-apitools
+oauthlib==3.1.0           # via requests-oauthlib
+opt-einsum==3.1.0         # via tensorflow
+pandas==0.25.3            # via tensorflow-data-validation
+pandocfilters==1.4.2      # via nbconvert
+parso==0.6.1              # via jedi
+pbr==5.4.4                # via mock
+pexpect==4.8.0            # via ipython
+pickleshare==0.7.5        # via ipython
+prometheus-client==0.7.1  # via notebook
+prompt-toolkit==2.0.10    # via ipython, jupyter-console
+protobuf==3.11.3          # via apache-beam, google-api-core, google-cloud-bigquery, googleapis-common-protos, ml-metadata, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-metadata, tensorflow-model-analysis, tensorflow-serving-api, tensorflow-transform, tfx, tfx-bsl
+ptyprocess==0.6.0         # via pexpect, terminado
+pyarrow==0.15.1           # via apache-beam, tensorflow-data-validation, tfx, tfx-bsl
+pyasn1-modules==0.2.8     # via google-auth, oauth2client
+pyasn1==0.4.8             # via oauth2client, pyasn1-modules, rsa
+pydot==1.4.1              # via apache-beam, tensorflow-transform
+pygments==2.5.2           # via ipython, jupyter-console, nbconvert, qtconsole
+pymongo==3.10.1           # via apache-beam
+pyparsing==2.4.6          # via pydot
+pyrsistent==0.15.7        # via jsonschema
+python-dateutil==2.8.1    # via apache-beam, jupyter-client, pandas
+pytz==2019.3              # via apache-beam, google-api-core, pandas
+pyyaml==3.13              # via tfx
+pyzmq==19.0.0             # via jupyter-client, notebook
+qtconsole==4.6.0          # via jupyter
+requests-oauthlib==1.3.0  # via google-auth-oauthlib
+requests==2.23.0          # via docker, google-api-core, hdfs, requests-oauthlib, tensorboard
+rsa==4.0                  # via google-auth, oauth2client
+scikit-learn==0.21.3      # via tensorflow-data-validation
+scipy==1.4.1              # via scikit-learn, tensorflow, tensorflow-model-analysis
+send2trash==1.5.0         # via notebook
+six==1.14.0               # via absl-py, bleach, docker, fasteners, google-api-core, google-api-python-client, google-apitools, google-auth, google-pasta, google-resumable-media, grpcio, h5py, hdfs, jsonschema, keras-preprocessing, ml-metadata, mock, oauth2client, prompt-toolkit, protobuf, pyarrow, pyrsistent, python-dateutil, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx, tfx-bsl, traitlets, websocket-client
+tensorboard==2.1.0        # via tensorflow
+tensorflow-data-validation==0.21.2  # via tfx
+tensorflow-estimator==2.1.0  # via tensorflow
+tensorflow-metadata==0.21.1  # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx-bsl
+tensorflow-model-analysis==0.21.3  # via tfx
+tensorflow-serving-api==2.1.0  # via tfx, tfx-bsl
+tensorflow-transform==0.21.0  # via tensorflow-data-validation, tfx
+tensorflow==2.1.0         # via ml-metadata, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-serving-api, tensorflow-transform, tfx, tfx-bsl
+termcolor==1.1.0          # via tensorflow
+terminado==0.8.3          # via notebook
+testpath==0.4.4           # via nbconvert
+tfx-bsl==0.21.2           # via tensorflow-data-validation, tensorflow-model-analysis, tensorflow-transform, tfx
+tfx==0.21.0               # via -r requirements.in (line 1)
+tornado==6.0.3            # via ipykernel, jupyter-client, notebook, terminado
+traitlets==4.3.3          # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, nbconvert, nbformat, notebook, qtconsole
+uritemplate==3.0.1        # via google-api-python-client
+urllib3==1.25.8           # via requests
+wcwidth==0.1.8            # via prompt-toolkit
+webencodings==0.5.1       # via bleach
+websocket-client==0.57.0  # via docker
+werkzeug==1.0.0           # via tensorboard
+wheel==0.34.2             # via tensorboard, tensorflow
+widgetsnbextension==3.5.1  # via ipywidgets
+wrapt==1.12.0             # via tensorflow
+zipp==1.2.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/backend/update_requirements.sh b/backend/update_requirements.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euo pipefail
+IMAGE=${1:-"python:3.5"}
+exec docker run -v "$PWD:/src" -w /src --rm "$IMAGE" bash \
+  -c 'pip3 install pip-tools==4.5.0 > /dev/null 2>&1 && pip-compile -v requirements.in'
diff --git a/test/sample-test/Dockerfile b/test/sample-test/Dockerfile
@@ -6,21 +6,8 @@ RUN apt-get update -y
 RUN apt-get install --no-install-recommends -y -q libssl-dev libffi-dev wget ssh
 RUN apt-get install --no-install-recommends -y -q default-jre default-jdk python3-setuptools python3-dev gcc
 
-RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
-
-RUN pip3 install wheel
-RUN pip3 install junit-xml
-RUN pip3 install kubernetes==9.0.0
-RUN pip3 install minio
-RUN pip3 install setuptools==40.5.0
-RUN pip3 install papermill==1.0.1
-RUN pip3 install ipykernel==5.1.0
-RUN pip3 install google-api-python-client==1.7.0
-RUN pip3 install google-cloud-storage==1.17.0
-RUN pip3 install fire==0.2.1
-RUN pip3 install yamale==2.0
-RUN pip3 install tfx==0.21.0
-
+COPY ./test/sample-test/requirements.txt /python/src/github.com/kubeflow/pipelines/test/sample-test/requirements.txt
+RUN pip3 install -r /python/src/github.com/kubeflow/pipelines/test/sample-test/requirements.txt
 # Install python client, including DSL compiler.
 COPY ./sdk/python /sdk/python
 RUN cd /sdk/python && python3 setup.py install

diff --git a/test/sample-test/requirements.in b/test/sample-test/requirements.in
@@ -0,0 +1,12 @@
+wheel
+junit-xml
+kubernetes==9.0.0
+minio
+setuptools
+papermill==1.0.1
+ipykernel==5.1.0
+google-api-python-client==1.7.8
+google-cloud-storage==1.17.0
+fire==0.2.1
+yamale==2.0
+tfx==0.21.0