From 6dc54833b0cab2bfcab1dfee91f01a96ebe99ad9 Mon Sep 17 00:00:00 2001 From: Anik Bhattacharjee Date: Tue, 24 Oct 2023 18:57:43 -0400 Subject: [PATCH] stand up registry for e2e Signed-off-by: Anik Bhattacharjee --- Makefile | 7 +- test/e2e/e2e_suite_test.go | 15 +--- .../catalogd_controller_patch.yaml | 18 +++++ test/tools/imageregistry/imagebuilder.yaml | 41 ++++++++++ test/tools/imageregistry/imgreg.yaml | 75 +++++++++++++++++++ test/tools/imageregistry/registry.sh | 69 +++++++++++++++++ testdata/catalogs/test-catalog.Dockerfile | 4 + testdata/catalogs/test-catalog/.indexignore | 2 + 8 files changed, 215 insertions(+), 16 deletions(-) create mode 100644 test/tools/imageregistry/catalogd_controller_patch.yaml create mode 100644 test/tools/imageregistry/imagebuilder.yaml create mode 100644 test/tools/imageregistry/imgreg.yaml create mode 100755 test/tools/imageregistry/registry.sh create mode 100644 testdata/catalogs/test-catalog/.indexignore diff --git a/Makefile b/Makefile index 2964debff..1fe85a503 100644 --- a/Makefile +++ b/Makefile @@ -118,11 +118,14 @@ UNIT_TEST_DIRS=$(shell go list ./... | grep -v /test/) test-unit: $(SETUP_ENVTEST) #HELP Run the unit tests eval $$($(SETUP_ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out +image-registry: ## Setup in-cluster image registry + ./test/tools/imageregistry/registry.sh + .PHONY: test-e2e test-e2e: KIND_CLUSTER_NAME=operator-controller-e2e test-e2e: KUSTOMIZE_BUILD_DIR=config/e2e test-e2e: GO_BUILD_FLAGS=-cover -test-e2e: run kind-load-test-artifacts e2e e2e-coverage undeploy kind-clean #HELP Run e2e test suite on local kind cluster +test-e2e: run image-registry kind-load-test-artifacts e2e e2e-coverage undeploy kind-clean #HELP Run e2e test suite on local kind cluster .PHONY: operator-developer-e2e operator-developer-e2e: KIND_CLUSTER_NAME=operator-controller-op-dev-e2e #EXHELP Run operator-developer e2e on local kind cluster @@ -161,7 +164,6 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images $(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 $(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0 - $(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog:e2e $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 --name $(KIND_CLUSTER_NAME) @@ -170,7 +172,6 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 --name $(KIND_CLUSTER_NAME) $(KIND) load docker-image localhost/testdata/bundles/plain-v0/plain:v0.1.0 --name $(KIND_CLUSTER_NAME) - $(KIND) load docker-image localhost/testdata/catalogs/test-catalog:e2e --name $(KIND_CLUSTER_NAME) .PHONY: deploy-local-registry deploy-local-registry: #EXHELP Deploy local registry. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 16d9d2804..9acdf1dbf 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -13,7 +13,6 @@ import ( "k8s.io/utils/env" "k8s.io/apimachinery/pkg/api/errors" - apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -35,11 +34,11 @@ var ( const ( testCatalogRefEnvVar = "TEST_CATALOG_IMAGE" - testCatalogRefDefault = "localhost/testdata/catalogs/test-catalog:e2e" + testCatalogRefDefault = "docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e" testCatalogName = "test-catalog" ) -// returns the image reference for the test, checking for environment variable substitution, with a default of localhost/testdata/catalogs/test-catalog:e2e +// returns the image reference for the test, checking for environment variable substitution, with a default of docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e func getCatalogImageRef() string { if s := os.Getenv(testCatalogRefEnvVar); s != "" { return s @@ -78,16 +77,6 @@ var _ = BeforeSuite(func() { operatorCatalog, err = createTestCatalog(ctx, testCatalogName, getCatalogImageRef()) Expect(err).ToNot(HaveOccurred()) - // TODO: REMOVE THIS. This should not be necessary if we have idiomatic APIs. Kubernetes is supposed - // to be eventually consistent. Waits for preconditions like this hide bugs. - Eventually(func(g Gomega) { - err := c.Get(ctx, types.NamespacedName{Name: operatorCatalog.Name}, operatorCatalog) - g.Expect(err).ToNot(HaveOccurred()) - cond := apimeta.FindStatusCondition(operatorCatalog.Status.Conditions, catalogd.TypeUnpacked) - g.Expect(cond).ToNot(BeNil()) - g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) - g.Expect(cond.Reason).To(Equal(catalogd.ReasonUnpackSuccessful)) - }).Should(Succeed()) }) var _ = AfterSuite(func() { diff --git a/test/tools/imageregistry/catalogd_controller_patch.yaml b/test/tools/imageregistry/catalogd_controller_patch.yaml new file mode 100644 index 000000000..a07471b13 --- /dev/null +++ b/test/tools/imageregistry/catalogd_controller_patch.yaml @@ -0,0 +1,18 @@ +spec: + template: + spec: + containers: + - name: manager + volumeMounts: + - mountPath: /etc/ssl/certs/ + name: ca-certs + readOnly: true + volumes: + - name: ca-certs + configMap: + name: docker-registry.operator-controller-e2e.svc + defaultMode: 0644 + optional: false + items: + - key: ca-certificates.crt + path: ca-certificates.crt \ No newline at end of file diff --git a/test/tools/imageregistry/imagebuilder.yaml b/test/tools/imageregistry/imagebuilder.yaml new file mode 100644 index 000000000..8d2fe83f5 --- /dev/null +++ b/test/tools/imageregistry/imagebuilder.yaml @@ -0,0 +1,41 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: kaniko + namespace: operator-controller-e2e +spec: + template: + spec: + containers: + - name: kaniko + image: gcr.io/kaniko-project/executor:latest + args: ["--dockerfile=/workspace/test-catalog.Dockerfile", + "--context=/workspace/", + "--destination=docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e"] + volumeMounts: + - name: certs + mountPath: /etc/ssl/certs/ + readOnly: true + - name: dockerfile + mountPath: /workspace/ + - name: build-contents + mountPath: /workspace/test-catalog/ + restartPolicy: Never + volumes: + - name: dockerfile + configMap: + name: operator-controller-e2e.dockerfile + items: + - key: test-catalog.Dockerfile + path: test-catalog.Dockerfile + - name: build-contents + configMap: + name: operator-controller-e2e.build-contents + - name: certs + configMap: + name: regcerts + defaultMode: 0644 + optional: false + items: + - key: ca-certificates.crt + path: ca-certificates.crt diff --git a/test/tools/imageregistry/imgreg.yaml b/test/tools/imageregistry/imgreg.yaml new file mode 100644 index 000000000..dacb58a3d --- /dev/null +++ b/test/tools/imageregistry/imgreg.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: operator-controller-e2e +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: operator-controller-e2e +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: operator-controller-e2e-registry + namespace: operator-controller-e2e +spec: + secretName: operator-controller-e2e-registry + isCA: true + dnsNames: + - docker-registry.operator-controller-e2e.svc + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: docker-registry + namespace: operator-controller-e2e + labels: + app: registry +spec: + replicas: 1 + selector: + matchLabels: + app: registry + template: + metadata: + labels: + app: registry + spec: + containers: + - name: registry + image: registry:2 + volumeMounts: + - name: certs-vol + mountPath: "/certs" + env: + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: "/certs/tls.crt" + - name: REGISTRY_HTTP_TLS_KEY + value: "/certs/tls.key" + volumes: + - name: certs-vol + secret: + secretName: operator-controller-e2e-registry +--- +apiVersion: v1 +kind: Service +metadata: + name: docker-registry + namespace: operator-controller-e2e +spec: + selector: + app: registry + ports: + - port: 5000 + targetPort: 5000 diff --git a/test/tools/imageregistry/registry.sh b/test/tools/imageregistry/registry.sh new file mode 100755 index 000000000..e89e969e0 --- /dev/null +++ b/test/tools/imageregistry/registry.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -e + +# registry.sh will create an in-cluster image registry useful for end-to-end testing +# of catalogd's unpacking process. It does a few things: +# 1. Installs cert-manager for creating a self-signed certificate for the image registry +# 2. Creates all the resources necessary for deploying the image registry in the operator-controller-e2e namespace +# 3. Creates a ConfigMap containing the CA cert for the image registry to be used by the catalogd-controller-manager +# 4. Creates a ConfigMap containing the CA cert for the image registry to be used by the kaniko pod +# 5. Creates ConfigMaps containing the test catalog + Dockerfile to be mounted to the kaniko pod +# 6. Waits for kaniko pod to have Phase == Succeeded, indicating the test catalog image has been built + pushed +# to the test image registry +# Usage: +# registry.sh + +# Install cert-manager +kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/v1.13.1/cert-manager.yaml" +kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s + +# create the image registry with all the certs +kubectl apply -f test/tools/imageregistry/imgreg.yaml +kubectl wait -n operator-controller-e2e --for=condition=Available deployment/docker-registry --timeout=60s + +# get cert value +certval=$(kubectl -n operator-controller-e2e get secret/operator-controller-e2e-registry -o=jsonpath='{.data.ca\.crt}' | base64 -d | sed 's/^/ /') + +kubectl apply -f - << EOF +apiVersion: v1 +kind: Namespace +metadata: + name: catalogd-system +EOF +# create a ConfigMap that contains the CA certs for the image registry +# This one is created for the catalogd-controller-manager +kubectl apply -f - << EOF +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: catalogd-system + name: docker-registry.operator-controller-e2e.svc +data: + "ca-certificates.crt": | +${certval} +EOF + +# create a ConfigMap that contains the CA certs for the image registry +# this one is created for the kaniko pod +kubectl apply -f - << EOF +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: operator-controller-e2e + name: regcerts +data: + "ca-certificates.crt": | +${certval} +EOF + +# Load the testdata onto the cluster as a configmap so it can be used with kaniko +kubectl create configmap -n operator-controller-e2e --from-file=testdata/catalogs/test-catalog.Dockerfile operator-controller-e2e.dockerfile +kubectl create configmap -n operator-controller-e2e --from-file=testdata/catalogs/test-catalog operator-controller-e2e.build-contents + +# Create the kaniko pod to build the test image and push it to the test registry. +kubectl apply -f test/tools/imageregistry/imagebuilder.yaml +kubectl wait --for=condition=Complete -n operator-controller-e2e jobs/kaniko --timeout=60s + +kubectl patch deployment catalogd-controller-manager -n catalogd-system --patch-file test/tools/imageregistry/catalogd_controller_patch.yaml +kubectl rollout status deployment/catalogd-controller-manager -n catalogd-system --timeout=540s diff --git a/testdata/catalogs/test-catalog.Dockerfile b/testdata/catalogs/test-catalog.Dockerfile index ea1030964..efe922e05 100644 --- a/testdata/catalogs/test-catalog.Dockerfile +++ b/testdata/catalogs/test-catalog.Dockerfile @@ -1,2 +1,6 @@ FROM scratch ADD test-catalog /configs + +# Set DC-specific label for the location of the DC root directory +# in the image +LABEL operators.operatorframework.io.index.configs.v1=/configs diff --git a/testdata/catalogs/test-catalog/.indexignore b/testdata/catalogs/test-catalog/.indexignore new file mode 100644 index 000000000..699fa6d33 --- /dev/null +++ b/testdata/catalogs/test-catalog/.indexignore @@ -0,0 +1,2 @@ +/expected_all.json +..*