Use kubeflow-ci as the test cluster (#411)

* use kubeflow-ci as the test cluster * change cluster to kubeflow-ci * remove param * also change bucket * use github token in checkout * fix * fix * fix * fix * fix * fix * fix
kubeflow · Mar 15, 2018 · 08512aa · 08512aa
1 parent 48098e6
commit 08512aa
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 104 deletions.
diff --git a/components/k8s-model-server/images/releaser/components/params.libsonnet b/components/k8s-model-server/images/releaser/components/params.libsonnet
@@ -7,17 +7,10 @@
     // Component-level parameters, defined initially from 'ks prototype use ...'
     // Each object below should correspond to a component in the components/ directory
     workflows: {
-      bucket: "mlkube-testing_temp",
       commit: "master",
       name: "new9",
       namespace: "kubeflow-test-infra",
       prow_env: "REPO_OWNER=kubeflow,REPO_NAME=kubeflow,PULL_BASE_SHA=master",
-      serving_image: "gcr.io/kubeflow-ci/model-server",
-      testing_image: "gcr.io/mlkube-testing/test-worker:latest",
-      tf_testing_image: "gcr.io/kubeflow-ci/tf-test-worker:1.0",
-      project: "mlkube-testing",
-      cluster: "kubeflow-testing",
-      zone: "us-east1-d",
     },
     // Test deploying a GPU model.
     gpu_model: {

diff --git a/components/k8s-model-server/images/releaser/components/workflows.libsonnet b/components/k8s-model-server/images/releaser/components/workflows.libsonnet
@@ -24,7 +24,7 @@
   //
   // TODO(jlewi): Use camelCase consistently.
   defaultParams:: {
-    bucket: "mlkube-testing_temp",
+    bucket: "kubeflow-ci_temp",
     commit: "master",
     // Name of the secret containing GCP credentials.
     gcpCredentialsSecretName: "kubeflow-testing-credentials",
@@ -38,7 +38,7 @@
     // The default image to use for the steps in the Argo workflow.
     testing_image: "gcr.io/mlkube-testing/kubeflow-testing",
     tf_testing_image: "gcr.io/kubeflow-ci/tf-test-worker:1.0",
-    project: "mlkube-testing",
+    project: "kubeflow-ci",
     cluster: "kubeflow-testing",
     zone: "us-east1-d",
     build_image: false,
@@ -283,28 +283,15 @@
                 }],
               ],
             },
-            {
-              name: "checkout",
-              container: {
-                command: [
-                  "/usr/local/bin/checkout.sh",
-                ],
-                args: [
-                  srcRootDir,
-                ],
-                env: prow_env + [{
-                  name: "EXTRA_REPOS",
-                  value: "kubeflow/testing@HEAD",
-                }],
-                image: testing_image,
-                volumeMounts: [
-                  {
-                    name: dataVolume,
-                    mountPath: mountPath,
-                  },
-                ],
-              },
-            },  // checkout
+            buildTemplate(
+              "checkout",
+              ["/usr/local/bin/checkout.sh", srcRootDir],
+              [{
+                name: "EXTRA_REPOS",
+                value: "kubeflow/testing@HEAD",
+              }],
+              [], // no sidecars
+            ),
 
             buildImageTemplate("build-tf-serving-cpu", "Dockerfile.cpu", cpuImage),
 

diff --git a/testing/workflows/components/params.libsonnet b/testing/workflows/components/params.libsonnet
@@ -7,7 +7,7 @@
     // Component-level parameters, defined initially from 'ks prototype use ...'
     // Each object below should correspond to a component in the components/ directory
     workflows: {
-      bucket: "mlkube-testing_temp",
+      bucket: "kubeflow-ci_temp",
       name: "jlewi-kubeflow-kubeflow-presubmit-test-227-643b",
       namespace: "kubeflow-test-infra",
       prow: "JOB_NAME=kubeflow-presubmit-test,JOB_TYPE=presubmit,PULL_NUMBER=209,REPO_NAME=kubeflow,REPO_OWNER=kubeflow,BUILD_NUMBER=997a",

diff --git a/testing/workflows/components/workflows.libsonnet b/testing/workflows/components/workflows.libsonnet
@@ -48,56 +48,56 @@
       local tfOperatorRoot = srcRootDir + "/kubeflow/tf-operator";
       local tfOperatorPy = tfOperatorRoot;
 
-      local project = "mlkube-testing";
+      local project = "kubeflow-ci";
       // GKE cluster to use
       local cluster = "kubeflow-testing";
       local zone = "us-east1-d";
-      {
-        // Build an Argo template to execute a particular command.
-        // step_name: Name for the template
-        // command: List to pass as the container command.
-        buildTemplate(step_name, command):: {
-          name: step_name,
-          container: {
-            command: command,
-            image: image,
-            env: [
-              {
-                // Add the source directories to the python path.
-                name: "PYTHONPATH",
-                value: kubeflowPy + ":" + kubeflowTestingPy + ":" + tfOperatorPy,
-              },
-              {
-                name: "GOOGLE_APPLICATION_CREDENTIALS",
-                value: "/secret/gcp-credentials/key.json",
-              },
-              {
-                name: "GITHUB_TOKEN",
-                valueFrom: {
-                  secretKeyRef: {
-                    name: "github-token",
-                    key: "github_token",
-                  },
+      // Build an Argo template to execute a particular command.
+      // step_name: Name for the template
+      // command: List to pass as the container command.
+      local buildTemplate(step_name, command, env_vars=[], sidecars=[]) = {
+        name: step_name,
+        container: {
+          command: command,
+          image: image,
+          env: [
+            {
+              // Add the source directories to the python path.
+              name: "PYTHONPATH",
+              value: kubeflowPy + ":" + kubeflowTestingPy + ":" + tfOperatorPy,
+            },
+            {
+              name: "GOOGLE_APPLICATION_CREDENTIALS",
+              value: "/secret/gcp-credentials/key.json",
+            },
+            {
+              name: "GITHUB_TOKEN",
+              valueFrom: {
+                secretKeyRef: {
+                  name: "github-token",
+                  key: "github_token",
                 },
               },
-            ] + prow_env,
-            volumeMounts: [
-              {
-                name: dataVolume,
-                mountPath: mountPath,
-              },
-              {
-                name: "github-token",
-                mountPath: "/secret/github-token",
-              },
-              {
-                name: "gcp-credentials",
-                mountPath: "/secret/gcp-credentials",
-              },
-            ],
-          },
-        },  // buildTemplate
-
+            },
+          ] + prow_env + env_vars,
+          volumeMounts: [
+            {
+              name: dataVolume,
+              mountPath: mountPath,
+            },
+            {
+              name: "github-token",
+              mountPath: "/secret/github-token",
+            },
+            {
+              name: "gcp-credentials",
+              mountPath: "/secret/gcp-credentials",
+            },
+          ],
+        },
+        sidecars: sidecars,
+      };  // buildTemplate
+      {
         apiVersion: "argoproj.io/v1alpha1",
         kind: "Workflow",
         metadata: {
@@ -170,29 +170,16 @@
                 }],
               ],
             },
-            {
-              name: "checkout",
-              container: {
-                command: [
-                  "/usr/local/bin/checkout.sh",
-                ],
-                args: [
-                  srcRootDir,
-                ],
-                env: prow_env + [{
-                  name: "EXTRA_REPOS",
-                  value: "kubeflow/tf-operator@HEAD;kubeflow/testing@HEAD",
-                }],
-                image: image,
-                volumeMounts: [
-                  {
-                    name: dataVolume,
-                    mountPath: mountPath,
-                  },
-                ],
-              },
-            },  // checkout
-            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("setup", [
+            buildTemplate(
+              "checkout",
+              ["/usr/local/bin/checkout.sh", srcRootDir],
+              [{
+                name: "EXTRA_REPOS",
+                value: "kubeflow/tf-operator@HEAD;kubeflow/testing@HEAD",
+              }],
+              [], // no sidecars
+            ),
+            buildTemplate("setup", [
               "python",
               "-m",
               "testing.test_deploy",
@@ -204,7 +191,7 @@
               "--artifacts_dir=" + artifactsDir,
               "setup",
             ]),  // setup
-            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("teardown", [
+            buildTemplate("teardown", [
               "python",
               "-m",
               "testing.test_deploy",
@@ -216,23 +203,23 @@
               "--artifacts_dir=" + artifactsDir,
               "teardown",
             ]),  // teardown
-            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("create-pr-symlink", [
+            buildTemplate("create-pr-symlink", [
               "python",
               "-m",
               "kubeflow.testing.prow_artifacts",
               "--artifacts_dir=" + outputDir,
               "create_pr_symlink",
               "--bucket=" + bucket,
             ]),  // create-pr-symlink
-            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("copy-artifacts", [
+            buildTemplate("copy-artifacts", [
               "python",
               "-m",
               "kubeflow.testing.prow_artifacts",
               "--artifacts_dir=" + outputDir,
               "copy_artifacts",
               "--bucket=" + bucket,
             ]),  // copy-artifacts
-            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("tfjob-test", [
+            buildTemplate("tfjob-test", [
               "python",
               "-m",
               "py.test_runner",