From 353ab0b2acc60f57ce773077d9fdb593a6b34ab1 Mon Sep 17 00:00:00 2001
From: IronPan <yangpa@google.com>
Date: Mon, 17 Dec 2018 15:26:23 -0800
Subject: [PATCH] Parameterize mysql and minio image (#551)

* add job to load sample

* debug

* add job to ks list

* update permission

* update permission

* parameterize deployment

* update param
---
 pipeline/pipeline/all.libsonnet               |  6 ++++--
 pipeline/pipeline/minio.libsonnet             | 18 +++++++++---------
 pipeline/pipeline/mysql.libsonnet             | 17 +++++++++--------
 pipeline/pipeline/prototypes/pipeline.jsonnet |  2 ++
 4 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/pipeline/pipeline/all.libsonnet b/pipeline/pipeline/all.libsonnet
index bfed17e44c0..74d6a0287a3 100644
--- a/pipeline/pipeline/all.libsonnet
+++ b/pipeline/pipeline/all.libsonnet
@@ -17,6 +17,8 @@
     local scheduledWorkflowImage = params.scheduledWorkflowImage,
     local persistenceAgentImage = params.persistenceAgentImage,
     local uiImage = params.uiImage,
+    local mysqlImage = params.mysqlImage,
+    local minioImage = params.minioImage,
     local deployArgo = params.deployArgo,
     local reportUsage = params.reportUsage,
     local usageId = params.usageId,
@@ -26,8 +28,8 @@
     argo:: if (deployArgo == true) || (deployArgo == "true") then
       argo.parts(namespace).all
     else [],
-    all:: minio.parts(namespace).all +
-          mysql.parts(namespace).all +
+    all:: minio.parts(namespace, minioImage).all +
+          mysql.parts(namespace, mysqlImage).all +
           pipeline_apiserver.all(namespace, apiImage) +
           pipeline_scheduledworkflow.all(namespace, scheduledWorkflowImage) +
           pipeline_persistenceagent.all(namespace, persistenceAgentImage) +
diff --git a/pipeline/pipeline/minio.libsonnet b/pipeline/pipeline/minio.libsonnet
index 788d37f6b32..3c1ebbcaa48 100644
--- a/pipeline/pipeline/minio.libsonnet
+++ b/pipeline/pipeline/minio.libsonnet
@@ -1,12 +1,12 @@
 {
-  parts(namespace):: {
-    all:: [
-      $.parts(namespace).pvc,
-      $.parts(namespace).service,
-      $.parts(namespace).deploy,
-      $.parts(namespace).secret,
-    ],
+  all(namespace, minioImage):: [
+    $.parts(namespace).pvc,
+    $.parts(namespace).service,
+    $.parts(namespace).deploy(minioImage),
+    $.parts(namespace).secret,
+  ],
 
+  parts(namespace):: {
     pvc: {
       apiVersion: "v1",
       kind: "PersistentVolumeClaim",
@@ -50,7 +50,7 @@
       },
     },  //service
 
-    deploy: {
+    deploy(image): {
       apiVersion: "apps/v1beta1",
       kind: "Deployment",
       metadata: {
@@ -85,7 +85,7 @@
                     mountPath: "/data",
                   },
                 ],
-                image: "minio/minio:RELEASE.2018-02-09T22-40-05Z",
+                image: image,
                 args: [
                   "server",
                   "/data",
diff --git a/pipeline/pipeline/mysql.libsonnet b/pipeline/pipeline/mysql.libsonnet
index 79a4b624353..1cd55aa4ad8 100644
--- a/pipeline/pipeline/mysql.libsonnet
+++ b/pipeline/pipeline/mysql.libsonnet
@@ -1,11 +1,10 @@
 {
+  all(namespace, mysqlImage):: [
+    $.parts(namespace).pvc,
+    $.parts(namespace).service,
+    $.parts(namespace).deploy(mysqlImage),
+  ],
   parts(namespace):: {
-    all:: [
-      $.parts(namespace).pvc,
-      $.parts(namespace).service,
-      $.parts(namespace).deploy,
-    ],
-
     pvc: {
       apiVersion: "v1",
       kind: "PersistentVolumeClaim",
@@ -47,7 +46,7 @@
       },
     },  //service
 
-    deploy: {
+    deploy(image): {
       apiVersion: "apps/v1beta2",
       kind: "Deployment",
       metadata: {
@@ -72,7 +71,9 @@
           spec: {
             containers: [
               {
-                image: "mysql:5.6",
+                // Before bumping the mysql version, make sure version is also available for
+                // Google Cloud SQL. As of Dec 2018, only mysql 5.6 or 5.7 is supported in Cloud SQL.
+                image: image,
                 name: "mysql",
                 env: [
                   {
diff --git a/pipeline/pipeline/prototypes/pipeline.jsonnet b/pipeline/pipeline/prototypes/pipeline.jsonnet
index 494714ea7af..5dd608172d9 100644
--- a/pipeline/pipeline/prototypes/pipeline.jsonnet
+++ b/pipeline/pipeline/prototypes/pipeline.jsonnet
@@ -9,6 +9,8 @@
 // @optionalParam uiImage string gcr.io/ml-pipeline/frontend:0.1.3 UI docker image
 // @optionalParam deployArgo string false flag to deploy argo
 // @optionalParam reportUsage string false flag to report usage
+// @optionalParam mysqlImage string mysql:5.6 mysql image
+// @optionalParam minioImage string minio/minio:RELEASE.2018-02-09T22-40-05Z minio image
 
 local k = import "k.libsonnet";
 local all = import "pipeline/pipeline/all.libsonnet";