test(gcp): cleanup blueprints - fix3 (kubeflow#799)

* test: fix cleanup blueprints script

* fix2

* fix3

* fix Makefile, update ACM repo, add dryrun=false

Makefile

@@ -11,13 +11,14 @@ TEKTON_INSTALLS=./tekton/templates/installs
 hydrate:
 	find $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy -type f -not -name namespace.yaml -exec rm {} ";"
 	rm -f $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy/tekton*
-	rm -f $(REPO_DIRS)/kf-ci-v1/namespaces/kf-ci/kf-ci	
+	rm -f $(REPO_DIRS)/kf-ci-v1/namespaces/kf-ci/kf-ci
 	kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy $(TEKTON_INSTALLS)/auto-deploy
 	kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/auto-deploy test-infra/auto-deploy/manifest
 	kustomize build -o $(REPO_DIRS)/kf-ci-v1/namespaces/kf-ci $(TEKTON_INSTALLS)/kf-ci
+	cd test-infra/cleanup && make hydrate
 
 # Make sure there are no nomos errors
-.PHONY: 
+.PHONY:
 acm-test:
 	nomos vet --no-api-server-check --path=acm-repos/kf-ci-management
 	nomos vet --no-api-server-check --path=acm-repos/kf-ci-v1
@@ -33,14 +34,14 @@ build-worker-image:
 set-worker-image:
 	kpt cfg set ./tekton test-image $(shell yq r ./images/latest_image.json builds[0].tag)
 
-update-worker-image: build-worker-image set-worker-image	
+update-worker-image: build-worker-image set-worker-image
 
 # This is a debug rule providing some sugar to hydrate and push the manifests and then wait for the
 # sync
 debug-push-and-run:
 	make hydrate && git add . && git commit -m "Latest" && git push jlewi
 	cd ./go/cmd/nomos-wait && go run .
-	kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml 
+	kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml
 
 # This is a debug rule providing some sugar for fast iteration during development
 # It might need to be customized for your usage.
@@ -52,4 +53,4 @@ debug-rebuild-and-run:
 	make update-worker-image
 	make hydrate && git add . && git commit -m "Latest" && git push jlewi
 	cd ./go/cmd/nomos-wait && go run .
-	kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml 
+	kubectl --context=kf-ci-v1 create -f ./tekton/runs/nb-test-run.yaml

acm-repos/kf-ci-v1/namespaces/auto-deploy/apps_v1_deployment_auto-deploy-server.yaml

@@ -136,7 +136,7 @@ spec:
       serviceAccount: default-editor
       volumes:
       - configMap:
-          name: auto-deploy-config-2778d8c6c8
+          name: auto-deploy-config-7f4chh5g64
         name: config
       - emptyDir: {}
         name: cache

acm-repos/kf-ci-v1/namespaces/auto-deploy/batch_v1beta1_cronjob_cleanup-ci-kubeflow-ci-deployment.yaml

@@ -41,7 +41,7 @@ spec:
           serviceAccountName: default-editor
           volumes:
           - configMap:
-              name: cleanup-config-dh2846bbg5
+              name: cleanup-config-556b6f2hdb
             name: cleanup-config
   schedule: 0 */2 * * *
   successfulJobsHistoryLimit: 3

acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml

@@ -57,6 +57,7 @@ spec:
     - auto-blueprints
     - --project=$(inputs.params.project)
     - --context=$(inputs.params.management-project)
+    - --dryrun=false
     env:
     - name: KUBECONFIG
       value: /workspace/kubeconfig

acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-2778d8c6c8.yaml → acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_auto-deploy-config-7f4chh5g64.yaml

@@ -132,12 +132,12 @@ data:
     \     type: git\n      params:\n        - name: revision\n          value: v1.1-branch\n
     \       - name: url\n          value: https://github.com/kubeflow/gcp-blueprints.git\n
     \ - name: testing-repo\n    resourceSpec:\n      type: git\n      params:\n        -
-    name: revision\n          value: master\n        - name: url\n          value:
+    name: revision\n          value: v1.1-branch\n        - name: url\n          value:
     https://github.com/kubeflow/testing.git\n  # Need to use a KSA with appropriate
-    GSA\n  serviceAccountName: default-editor\n  pipelineRef:\n    name: deploy-gcp-blueprint"
+    GSA\n  serviceAccountName: default-editor\n  pipelineRef:\n    name: deploy-gcp-blueprint\n"
 kind: ConfigMap
 metadata:
   labels:
     service: auto-deploy
-  name: auto-deploy-config-2778d8c6c8
+  name: auto-deploy-config-7f4chh5g64
   namespace: auto-deploy

acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-dh2846bbg5.yaml → acm-repos/kf-ci-v1/namespaces/auto-deploy/~g_v1_configmap_cleanup-config-556b6f2hdb.yaml

@@ -4,9 +4,11 @@ data:
     up the Kubeflow auto-deployed blueprints.\n#\napiVersion: tekton.dev/v1alpha1\nkind:
     PipelineRun\nmetadata:\n  generateName: cleanup-blueprints-\n  namespace: auto-deploy\nspec:\n
     \ # TODO(jlewi): Override any parameters?\n  #params: {}\n  resources:  \n  -
-    name: testing-repo\n    resourceSpec:\n      type: git\n      params:\n        #
-    TODO(jlewi): Switch to master on kubeflow/gcp-blueprints\n        - name: revision\n
-    \         value: master\n        - name: url\n          value: https://github.com/kubeflow/testing.git\n
+    name: testing-repo\n    resourceSpec:\n      type: git\n      params:\n        -
+    name: revision\n          value: master\n        - name: url\n          value:
+    https://github.com/kubeflow/testing.git\n        # Uncomment the following when
+    testing locally\n        # - name: revision\n        #   value: cleanup-blueprints\n
+    \       # - name: url\n        #   value: https://github.com/Bobgy/testing.git\n
     \ # Need to use a KSA with appropriate GSA\n  serviceAccountName: default-editor\n
     \ pipelineSpec:\n    params:\n    - name: management-cluster-name\n      type:
     string\n      description: The name of the management cluster. \n      default:
@@ -19,5 +21,5 @@ data:
     \       name: cleanup-kubeflow-ci\n        kind: namespaced  \n"
 kind: ConfigMap
 metadata:
-  name: cleanup-config-dh2846bbg5
+  name: cleanup-config-556b6f2hdb
   namespace: auto-deploy

acm-repos/kf-ci-v1/namespaces/kf-ci/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml

@@ -57,6 +57,7 @@ spec:
     - auto-blueprints
     - --project=$(inputs.params.project)
     - --context=$(inputs.params.management-project)
+    - --dryrun=false
     env:
     - name: KUBECONFIG
       value: /workspace/kubeconfig

py/kubeflow/testing/cleanup_blueprints.py

@@ -95,13 +95,16 @@ def _delete_blueprints(namespace, to_keep_names, context=None, dryrun=True):
   for kind in kinds:
     client = cnrm_clients.CnrmClientApi(api_client, kind)
 
-    selector = "{0}=true".format(AUTO_DEPLOY_LABEL)
+    # https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#set-based-requirement
+    # selects resources with GROUP_LABEL set
+    selector = GROUP_LABEL
     results = client.list_namespaced(namespace, label_selector=selector)
 
     for i in results.get("items"):
       name = i["metadata"]["name"]
+      kf_name = i["metadata"].get("labels", {}).get(NAME_LABEL, "")
 
-      if name in to_keep_names:
+      if kf_name in to_keep_names:
         to_keep[kind].append(name)
         continue
 
@@ -170,12 +173,12 @@ def auto_blueprints(project, context, dryrun=True, blueprints=None): # pylint: d
       # Use labels to identify auto-deployed instances
       auto_deploy_label = b["metadata"].get("labels", {}).get(AUTO_DEPLOY_LABEL,
                                                               "false")
-
-      is_auto_deploy = auto_deploy_label.lower() == "true"
-
-      if not is_auto_deploy:
-        logging.info("Skipping cluster %s; its missing the auto-deploy label",
-                     name)
+      blueprint_group = b["metadata"]["labels"].get(GROUP_LABEL, "unknown")
+      if blueprint_group == "unknown":
+        logging.info("Skipping cluster %s; its missing the %s label; it is not "
+                     "an auto-deployed instance",
+                     name, GROUP_LABEL)
+        continue
 
       # Tha name of blueprint
       kf_name = b["metadata"].get("labels", {}).get(NAME_LABEL, "")
@@ -199,12 +202,6 @@ def auto_blueprints(project, context, dryrun=True, blueprints=None): # pylint: d
 
       logging.info("Blueprint %s is auto deployed", kf_name)
 
-      blueprint_group = b["metadata"]["labels"].get(GROUP_LABEL, "unknown")
-
-      if blueprint_group == "unknown":
-        logging.warning("Blueprint %s was missing label %s", kf_name,
-                        GROUP_LABEL)
-
       if kf_name in auto_deployments[blueprint_group]:
         continue
 

tekton/templates/tasks/cleanup-kubeflow-ci.yaml

@@ -61,6 +61,7 @@ spec:
     - auto-blueprints
     - --project=$(inputs.params.project)
     - --context=$(inputs.params.management-project)
+    - --dryrun=false
     env:
     - name: KUBECONFIG
       value: /workspace/kubeconfig

test-infra/cleanup/cleanup-blueprints-pipeline.yaml

@@ -14,11 +14,15 @@ spec:
     resourceSpec:
       type: git
       params:
-        # TODO(jlewi): Switch to master on kubeflow/gcp-blueprints
         - name: revision
           value: master
         - name: url
           value: https://github.com/kubeflow/testing.git
+        # Uncomment the following when testing locally
+        # - name: revision
+        #   value: cleanup-blueprints
+        # - name: url
+        #   value: https://github.com/Bobgy/testing.git
   # Need to use a KSA with appropriate GSA
   serviceAccountName: default-editor
   pipelineSpec: