Initial batch of fixes to run E2E notebook tests against blueprint de…

…ployments. * Update get_kf_testing_cluster to work with blueprints * With GCP blueprints we won't have deployments so we need to list clusters by name and find cluster with matching regex. * Create an ACM repo for kf-ci-v1 cluster and hydrate it with the appropriate manifests for the auto-deploy and tektoncd namespaces. * kubeflow#677 is tracking using ACM with the cluster kf-ci-v1 * Note this PR is only using ACM to sync tekton resources; we still need to sync the rest of the auto-deployment infrastructure like the reconciler and webserver. * Remove ACM cluster selector; ACM complains because it isn't actually being applied. * Override notebook-test-task.yaml with nb-test-task.yaml. The latter should include the latest changes from Gabriel's PR. * When deploying the blueprint we need to do `kpt cfg set email` to set the email for the default profile. Without this change the deployment won't include the namespace we need to run the tests in GoogleCloudPlatform/kubeflow-distribution#42 is tracking CI for the blueprints.
jlewi · Jun 9, 2020 · dd29b6c · dd29b6c
1 parent 5e50f30
commit dd29b6c
Show file tree

Hide file tree

Showing 30 changed files with 1,296 additions and 263 deletions.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,14 @@
+REPO_DIRS=./acm-repos
+AUTO_DEPLOY_CONTEXT=kf-ci-v1
+
+#***************************************************************************************************
+
+TEKTON_INSTALLS=./tekton/templates/installs
+# Hydrate ACM repos
+.PHONY: hydrate
+hydrate:
+	rm $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy/tekton*
+	rm $(REPO_DIRS)/kf-ci-v1/namespaces/tektoncd/tekton*
+	kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy $(TEKTON_INSTALLS)/auto-deploy
+	kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/tektoncd $(TEKTON_INSTALLS)/tektoncd
+
diff --git a/acm-repos/kf-ci-v1/README.md b/acm-repos/kf-ci-v1/README.md
@@ -0,0 +1,5 @@
+# Anthos Configuration Management Directory
+
+This is the root directory for Anthos Configuration Management.
+
+See [our documentation](https://cloud.google.com/anthos-config-management/docs/repo) for how to use each subdirectory.
diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/namespace.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/namespace.yaml
@@ -0,0 +1,10 @@
+# This namespace is used to administer the issue-label-bot-dev project
+# It should only exist on the kf-ci-management cluster
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: auto-deploy
+  #annotations:
+    # TODO(jlewi): We should add a cluster selector to only select
+    # the KF ci managmenet cluster.
+    #configmanagement.gke.io/cluster-selector: kf-ci-tekt
diff --git a/...os/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_pipeline_deploy-gcp-blueprint.yaml b/...os/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_pipeline_deploy-gcp-blueprint.yaml
@@ -0,0 +1,42 @@
+apiVersion: tekton.dev/v1alpha1
+kind: Pipeline
+metadata:
+  name: deploy-gcp-blueprint
+  namespace: auto-deploy
+spec:
+  params:
+  - default: kf-vbp-{uid}
+    description: The name for the Kubeflow deployment
+    name: name
+    type: string
+  - default: kf-ci-management
+    description: The name of the management cluster.
+    name: management-cluster-name
+    type: string
+  - default: kubeflow-ci-deployment
+    description: The project to deploy into
+    name: project
+    type: string
+  resources:
+  - name: testing-repo
+    type: git
+  - name: blueprint-repo
+    type: git
+  tasks:
+  - name: deploy-gcp
+    params:
+    - name: name
+      value: $(params.name)
+    - name: project
+      value: $(params.project)
+    - name: management-cluster-name
+      value: $(params.management-cluster-name)
+    resources:
+      inputs:
+      - name: blueprint-repo
+        resource: blueprint-repo
+      - name: testing-repo
+        resource: testing-repo
+    taskRef:
+      kind: namespaced
+      name: deploy-gcp-blueprint
diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_pipeline_notebook-test.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_pipeline_notebook-test.yaml
@@ -0,0 +1,59 @@
+apiVersion: tekton.dev/v1alpha1
+kind: Pipeline
+metadata:
+  name: notebook-test
+  namespace: auto-deploy
+spec:
+  params:
+  - default: manual-testing
+    description: Test targe name, used to group test results in JUNIT.
+    name: test-target-name
+    type: string
+  - description: GCS bucket and directory artifacts will be uploaded to. Should be
+      in the form of 'gs://'
+    name: artifacts-gcs
+    type: string
+  - description: Relative path to the GCS artifacts will be uploaded to. Base path
+      is artifacts-gcs so the actual GCS blob will be artifacts-gcs/junit-path
+    name: junit-path
+    type: string
+  - description: Cluster pattern to run the notebook test. Default to be from master
+      branch.
+    name: testing-cluster-pattern
+    type: string
+  - description: Location to search for test clusters.
+    name: testing-cluster-location
+    type: string
+  - description: Directory to write outputs to in local FS.
+    name: output-workspace
+    type: string
+  resources:
+  - name: examples-repo
+    type: git
+  - name: testing-repo
+    type: git
+  tasks:
+  - name: mnist-gcp
+    params:
+    - name: notebook-path
+      value: kubeflow/examples/mnist/mnist_gcp.ipynb
+    - name: junit-path
+      value: $(params.output-workspace)/$(params.junit-path)/junit_mnist-gcp.xml
+    - name: test-target-name
+      value: $(params.test-target-name)
+    - name: output-workspace
+      value: $(params.output-workspace)
+    - name: artifacts-gcs
+      value: $(params.artifacts-gcs)
+    - name: testing-cluster-pattern
+      value: $(params.testing-cluster-pattern)
+    - name: testing-cluster-location
+      value: $(params.testing-cluster-location)
+    resources:
+      inputs:
+      - name: examples-repo
+        resource: examples-repo
+      - name: kf-testing-repo
+        resource: testing-repo
+    taskRef:
+      name: nb-tests
diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml
@@ -0,0 +1,66 @@
+apiVersion: tekton.dev/v1alpha1
+kind: Task
+metadata:
+  annotations:
+    sidecar.istio.io/inject: "false"
+  name: cleanup-kubeflow-ci
+  namespace: auto-deploy
+spec:
+  inputs:
+    params:
+    - default: kf-vbp-{uid}
+      description: The name for the Kubeflow deployment
+      name: name
+      type: string
+    - default: kubeflow-ci-deployment
+      description: The project to clean up.
+      name: project
+      type: string
+    - default: kf-ci-management
+      description: The name of the management cluster.
+      name: management-cluster-name
+      type: string
+    - default: kubeflow-ci
+      description: The project containing the management cluster
+      name: management-project
+      type: string
+    - default: us-central1
+      description: The location of the management cluster
+      name: management-location
+      type: string
+    resources:
+    - description: The GitHub repo containing kubeflow testing scripts
+      name: testing-repo
+      type: git
+  steps:
+  - command:
+    - python
+    - -m
+    - kubeflow.testing.create_context
+    - create
+    - --name=$(inputs.params.management-project)
+    - --project=$(inputs.params.management-project)
+    - --location=$(inputs.params.management-location)
+    - --cluster=$(inputs.params.management-cluster-name)
+    - --namespace=$(inputs.params.project)
+    env:
+    - name: KUBECONFIG
+      value: /workspace/kubeconfig
+    - name: PYTHONPATH
+      value: /workspace/$(inputs.resources.testing-repo.name)/py
+    image: gcr.io/kubeflow-ci/test-worker-py3@sha256:b679ce5d7edbcc373fd7d28c57454f4f22ae987f200f601252b6dcca1fd8823b
+    name: create-context
+  - command:
+    - python
+    - -m
+    - kubeflow.testing.cleanup_blueprints
+    - auto-blueprints
+    - --project=$(inputs.params.project)
+    - --context=$(inputs.params.management-project)
+    env:
+    - name: KUBECONFIG
+      value: /workspace/kubeconfig
+    - name: PYTHONPATH
+      value: /workspace/$(inputs.resources.testing-repo.name)/py
+    image: gcr.io/kubeflow-ci/test-worker-py3@sha256:b679ce5d7edbcc373fd7d28c57454f4f22ae987f200f601252b6dcca1fd8823b
+    name: cleanup-ci
diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_deploy-gcp-blueprint.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_deploy-gcp-blueprint.yaml
@@ -0,0 +1,79 @@
+apiVersion: tekton.dev/v1alpha1
+kind: Task
+metadata:
+  annotations:
+    sidecar.istio.io/inject: "false"
+  name: deploy-gcp-blueprint
+  namespace: auto-deploy
+spec:
+  inputs:
+    params:
+    - default: kf-vbp-{uid}
+      description: The name for the Kubeflow deployment
+      name: name
+      type: string
+    - default: kubeflow-ci-deployment
+      description: The project to deploy into.
+      name: project
+      type: string
+    - default: kf-ci-management
+      description: The name of the management cluster.
+      name: management-cluster-name
+      type: string
+    - default: kubeflow-ci
+      description: The project containing the management cluster
+      name: management-project
+      type: string
+    - default: us-central1
+      description: The location of the management cluster
+      name: management-location
+      type: string
+    resources:
+    - description: The GitHub repo containing kubeflow testing scripts
+      name: testing-repo
+      type: git
+    - description: The GitHub repo containing the blueprint
+      name: blueprint-repo
+      type: git
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /workspace/kubeconfig
+    image: gcr.io/kubeflow-ci/test-worker-py3@sha256:b679ce5d7edbcc373fd7d28c57454f4f22ae987f200f601252b6dcca1fd8823b
+  steps:
+  - command:
+    - /workspace/$(inputs.resources.blueprint-repo.name)/kubeflow/hack/create_context.sh
+    env:
+    - name: PROJECT
+      value: $(inputs.params.management-project)
+    - name: REGION
+      value: $(inputs.params.management-location)
+    - name: NAME
+      value: $(inputs.params.management-cluster-name)
+    - name: NAMESPACE
+      value: $(inputs.params.project)
+    name: get-credential
+  - command:
+    - python
+    - -m
+    - kubeflow.testing.create_kf_from_gcp_blueprint
+    - deploy
+    - --name=$(inputs.params.name)
+    - --blueprint-dir=/workspace/$(inputs.resources.blueprint-repo.name)/kubeflow
+    - --management-context=$(inputs.params.management-cluster-name)
+    - --labels-file=/etc/podinfo/labels
+    env:
+    - name: PYTHONPATH
+      value: /workspace/$(inputs.resources.testing-repo.name)/py
+    name: deploy-gcp
+    volumeMounts:
+    - mountPath: /etc/podinfo
+      name: podinfo
+    workingDir: /workspace/$(inputs.resources.blueprint-repo.name)/kubeflow
+  volumes:
+  - downwardAPI:
+      items:
+      - fieldRef:
+          fieldPath: metadata.labels
+        path: labels
+    name: podinfo
diff --git a/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_nb-tests.yaml b/acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_nb-tests.yaml
@@ -0,0 +1,108 @@
+apiVersion: tekton.dev/v1alpha1
+kind: Task
+metadata:
+  annotations:
+    sidecar.istio.io/inject: "false"
+  name: nb-tests
+  namespace: auto-deploy
+spec:
+  inputs:
+    params:
+    - description: Testing notebook location. Should be in the form of {REPO_OWNER}/{REPO}/path/to/notebook.ipynb
+      name: notebook-path
+      type: string
+    - description: Cluster pattern to run the notebook test. Default to be from master
+        branch.
+      name: testing-cluster-pattern
+      type: string
+    - description: Location to search for test clusters e.g. us-central1 or us-central1-f
+      name: testing-cluster-location
+      type: string
+    - description: GCS bucket and directory artifacts will be uploaded to. Should
+        be in the form of 'gs://'
+      name: artifacts-gcs
+      type: string
+    - description: Relative path to the GCS artifacts will be uploaded to. Base path
+        is artifacts-gcs so the actual GCS blob will be artifacts-gcs/junit-path
+      name: junit-path
+      type: string
+    - default: manual-testing
+      description: Test targe name, used to group test results in JUNIT.
+      name: test-target-name
+      type: string
+    - description: Directory to write outputs to in local FS.
+      name: output-workspace
+      type: string
+    resources:
+    - name: examples-repo
+      targetPath: src/kubeflow/examples
+      type: git
+    - name: kf-testing-repo
+      targetPath: src/kubeflow/testing
+      type: git
+  steps:
+  - args:
+    - -m
+    - kubeflow.testing.get_kf_testing_cluster
+    - --base=$(inputs.params.testing-cluster-pattern)
+    - --location=$(inputs.params.testing-cluster-location)
+    - get-credentials
+    command:
+    - python3
+    env:
+    - name: PYTHONPATH
+      value: /workspace/src/kubeflow/examples/py:/workspace/src/kubeflow/testing/py
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: /secret/gcp-credentials/key.json
+    image: gcr.io/kubeflow-ci/test-worker:latest
+    name: get-credential
+    volumeMounts:
+    - mountPath: /secret/gcp-credentials
+      name: gcp-credentials
+      readOnly: true
+  - env:
+    - name: PYTHONPATH
+      value: /workspace/src/kubeflow/examples/py:/workspace/src/kubeflow/testing/py
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: /secret/gcp-credentials/key.json
+    image: gcr.io/kubeflow-ci/test-worker:latest
+    name: run-notebook
+    script: |
+      #!/usr/bin/env bash
+      pytest run_notebook_test.py \
+        --log-cli-level=info \
+        --log-cli-format='%(levelname)s|%(asctime)s|%(pathname)s|%(lineno)d| %(message)s' \
+        --timeout=1800 \
+        --junitxml=$(inputs.params.junit-path) \
+        --notebook_path=$(inputs.params.notebook-path) \
+        --test-target-name=$(inputs.params.test-target-name) \
+        --artifacts-gcs=$(inputs.params.artifacts-gcs) \
+      || echo test finished.
+    volumeMounts:
+    - mountPath: /secret/gcp-credentials
+      name: gcp-credentials
+      readOnly: true
+    workingDir: /workspace/src/kubeflow/examples/py/kubeflow/examples/notebook_tests
+  - args:
+    - -m
+    - kubeflow.testing.tekton_client
+    - junit_parse_and_upload
+    - --artifacts_dir=$(inputs.params.output-workspace)
+    - --output_gcs=$(inputs.params.artifacts-gcs)
+    command:
+    - python
+    env:
+    - name: PYTHONPATH
+      value: /workspace/src/kubeflow/examples/py:/workspace/src/kubeflow/testing/py
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: /secret/gcp-credentials/key.json
+    image: gcr.io/kubeflow-ci/test-worker:latest
+    name: copy-artifacts
+    volumeMounts:
+    - mountPath: /secret/gcp-credentials
+      name: gcp-credentials
+      readOnly: true
+  volumes:
+  - name: gcp-credentials
+    secret:
+      secretName: gcp-credentials