Releasing component SDK

.cloudbuild.yaml

@@ -28,6 +28,20 @@ steps:
   id:   'copyPythonSDKToLatest'
   waitFor: ['preparePythonSDK']
 
+# Build the Python Component SDK
+- name: 'debian'
+  entrypoint: '/bin/bash'
+  args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python python-setuptools;cd /workspace/component_sdk/python;python setup.py sdist --format=gztar;cp dist/*.tar.gz /workspace/kfp-component.tar.gz']
+  id:   'preparePythonComponentSDK'
+- name: 'gcr.io/cloud-builders/gsutil'
+  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp-component.tar.gz']
+  id:   'copyPythonComponentSDK'
+  waitFor: ['preparePythonComponentSDK']
+- name: 'gcr.io/cloud-builders/gsutil'
+  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp-component.tar.gz']
+  id:   'copyPythonComponentSDKToLatest'
+  waitFor: ['preparePythonComponentSDK']
+
 # Build the pipeline system images
 - name: 'debian'
   entrypoint: '/bin/bash'
@@ -130,6 +144,12 @@ steps:
   args: ['-c', 'cd /workspace/components/resnet-cmle/containers/train && ./build.sh $PROJECT_ID $COMMIT_SHA']
   id: 'buildResnetCMLETrainComponent'
 
+# Build the Generic GCP component image
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: '/bin/bash'
+  args: ['-c', 'cd /workspace/components/gcp/container/ && ./build.sh $PROJECT_ID $COMMIT_SHA']
+  id: 'buildGcpGenericComponent'
+
 # Build the local pipeline component images
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: '/bin/bash'
@@ -183,6 +203,9 @@ images:
 - 'gcr.io/$PROJECT_ID/resnet-preprocess:$COMMIT_SHA'
 - 'gcr.io/$PROJECT_ID/resnet-train:$COMMIT_SHA'
 
+# Images for the GCP generic pipeline components
+- 'gcr.io/$PROJECT_ID/ml-pipeline-gcp:$COMMIT_SHA'
+
 # Images for the local components
 - 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'
 - 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA'

.release.cloudbuild.yaml

@@ -87,6 +87,19 @@ steps:
   id:   'copyPythonSDKToLatest'
   waitFor: ['copyPythonSDKLocal']
 
+# Copy the Python Component SDK
+- name: 'gcr.io/cloud-builders/gsutil'
+  args: ['cp', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp-component.tar.gz', '/workspace/']
+  id:   'copyPythonComponentSDKLocal'
+- name: 'gcr.io/cloud-builders/gsutil'
+  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://ml-pipeline/release/$TAG_NAME/kfp-component.tar.gz']
+  id:   'copyPythonComponentSDK'
+  waitFor: ['copyPythonComponentSDKLocal']
+- name: 'gcr.io/cloud-builders/gsutil'
+  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://ml-pipeline/release/latest/kfp-component.tar.gz']
+  id:   'copyPythonComponentSDKToLatest'
+  waitFor: ['copyPythonComponentSDKLocal']
+
 # Build the tagged samples
 - name: 'debian'
   entrypoint: '/bin/bash'

component_sdk/python/kfp_component/google/ml_engine/_train.py

@@ -16,10 +16,10 @@
 from ._create_job import create_job
 
 @decorators.SetParseFns(python_version=str, runtime_version=str)
-def train(project_id, python_module, package_uris, 
-    region, args=None, job_dir=None, python_version=None, 
-    runtime_version=None, training_input=None, job_id_prefix=None,
-    wait_interval=30):
+def train(project_id, python_module=None, package_uris=None, 
+    region=None, args=None, job_dir=None, python_version=None, 
+    runtime_version=None, master_image_uri=None, worker_image_uri=None, 
+    training_input=None, job_id_prefix=None, wait_interval=30):
     """Creates a MLEngine training job.
 
     Args:
@@ -44,6 +44,10 @@ def train(project_id, python_module, package_uris,
         runtime_version (str): Optional. The Cloud ML Engine runtime version 
             to use for training. If not set, Cloud ML Engine uses the 
             default stable version, 1.0. 
+        master_image_uri (str): The Docker image to run on the master replica. 
+            This image must be in Container Registry.
+        worker_image_uri (str): The Docker image to run on the worker replica. 
+            This image must be in Container Registry.
         training_input (dict): Input parameters to create a training job.
         job_id_prefix (str): the prefix of the generated job id.
         wait_interval (int): optional wait interval between calls
@@ -65,6 +69,14 @@ def train(project_id, python_module, package_uris,
         training_input['pythonVersion'] = python_version
     if runtime_version:
         training_input['runtimeVersion'] = runtime_version
+    if master_image_uri:
+        if 'masterConfig' not in training_input:
+            training_input['masterConfig'] = {}
+        training_input['masterConfig']['imageUri'] = master_image_uri
+    if worker_image_uri:
+        if 'workerConfig' not in training_input:
+            training_input['workerConfig'] = {}
+        training_input['workerConfig']['imageUri'] = worker_image_uri
     job = {
         'trainingInput': training_input
     }

component_sdk/python/tests/google/ml_engine/test__train.py

@@ -29,7 +29,8 @@ def test_train_succeed(self, mock_create_job):
             training_input={
                 'runtimeVersion': '1.10',
                 'pythonVersion': '2.7'
-            }, job_id_prefix='job-')
+            }, job_id_prefix='job-', master_image_uri='tensorflow:latest',
+            worker_image_uri='debian:latest')
 
         mock_create_job.assert_called_with('proj-1', {
             'trainingInput': {
@@ -39,6 +40,12 @@ def test_train_succeed(self, mock_create_job):
                 'args': ['arg-1', 'arg-2'],
                 'jobDir': 'gs://test/job/dir',
                 'runtimeVersion': '1.10',
-                'pythonVersion': '2.7'
+                'pythonVersion': '2.7',
+                'masterConfig': {
+                    'imageUri': 'tensorflow:latest'
+                },
+                'workerConfig': {
+                    'imageUri': 'debian:latest'
+                }
             }
         }, 'job-', 30)

components/gcp/container/Dockerfile

@@ -0,0 +1,21 @@
+# Copyright 2018 The Kubeflow Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.6
+
+ADD build /ml
+WORKDIR /ml
+RUN pip install .
+
+ENTRYPOINT ["python", "-u", "-m", "kfp_component.launcher"]

components/gcp/container/build_image.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -e
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p ./build
+rsync -arvp --exclude=.tox "../../../component_sdk/python"/ ./build/
+cp ../../license.sh ./build/
+cp ../../third_party_licenses.csv ./build/
+
+../../build_image.sh -l ml-pipeline-gcp "$@"
+rm -rf ./build

components/release.sh

@@ -43,6 +43,7 @@ images=(
   "resnet-train"
   "ml-pipeline-local-confusion-matrix"
   "ml-pipeline-local-roc"
+  "ml-pipeline-gcp"
 )
 
 COMMIT_SHA=$1