Skip to content

Commit

Permalink
stand up registry for e2e
Browse files Browse the repository at this point in the history
Signed-off-by: Anik Bhattacharjee <[email protected]>
  • Loading branch information
anik120 committed Oct 25, 2023
1 parent 2926297 commit 6dc5483
Show file tree
Hide file tree
Showing 8 changed files with 215 additions and 16 deletions.
7 changes: 4 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,14 @@ UNIT_TEST_DIRS=$(shell go list ./... | grep -v /test/)
test-unit: $(SETUP_ENVTEST) #HELP Run the unit tests
eval $$($(SETUP_ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out

image-registry: ## Setup in-cluster image registry
./test/tools/imageregistry/registry.sh

.PHONY: test-e2e
test-e2e: KIND_CLUSTER_NAME=operator-controller-e2e
test-e2e: KUSTOMIZE_BUILD_DIR=config/e2e
test-e2e: GO_BUILD_FLAGS=-cover
test-e2e: run kind-load-test-artifacts e2e e2e-coverage undeploy kind-clean #HELP Run e2e test suite on local kind cluster
test-e2e: run image-registry kind-load-test-artifacts e2e e2e-coverage undeploy kind-clean #HELP Run e2e test suite on local kind cluster

.PHONY: operator-developer-e2e
operator-developer-e2e: KIND_CLUSTER_NAME=operator-controller-op-dev-e2e #EXHELP Run operator-developer e2e on local kind cluster
Expand Down Expand Up @@ -161,7 +164,6 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images
$(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0
$(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog:e2e
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 --name $(KIND_CLUSTER_NAME)
Expand All @@ -170,7 +172,6 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/bundles/plain-v0/plain:v0.1.0 --name $(KIND_CLUSTER_NAME)
$(KIND) load docker-image localhost/testdata/catalogs/test-catalog:e2e --name $(KIND_CLUSTER_NAME)

.PHONY: deploy-local-registry
deploy-local-registry: #EXHELP Deploy local registry.
Expand Down
15 changes: 2 additions & 13 deletions test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"k8s.io/utils/env"

"k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
Expand All @@ -35,11 +34,11 @@ var (

const (
testCatalogRefEnvVar = "TEST_CATALOG_IMAGE"
testCatalogRefDefault = "localhost/testdata/catalogs/test-catalog:e2e"
testCatalogRefDefault = "docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e"
testCatalogName = "test-catalog"
)

// returns the image reference for the test, checking for environment variable substitution, with a default of localhost/testdata/catalogs/test-catalog:e2e
// returns the image reference for the test, checking for environment variable substitution, with a default of docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e
func getCatalogImageRef() string {
if s := os.Getenv(testCatalogRefEnvVar); s != "" {
return s
Expand Down Expand Up @@ -78,16 +77,6 @@ var _ = BeforeSuite(func() {
operatorCatalog, err = createTestCatalog(ctx, testCatalogName, getCatalogImageRef())
Expect(err).ToNot(HaveOccurred())

// TODO: REMOVE THIS. This should not be necessary if we have idiomatic APIs. Kubernetes is supposed
// to be eventually consistent. Waits for preconditions like this hide bugs.
Eventually(func(g Gomega) {
err := c.Get(ctx, types.NamespacedName{Name: operatorCatalog.Name}, operatorCatalog)
g.Expect(err).ToNot(HaveOccurred())
cond := apimeta.FindStatusCondition(operatorCatalog.Status.Conditions, catalogd.TypeUnpacked)
g.Expect(cond).ToNot(BeNil())
g.Expect(cond.Status).To(Equal(metav1.ConditionTrue))
g.Expect(cond.Reason).To(Equal(catalogd.ReasonUnpackSuccessful))
}).Should(Succeed())
})

var _ = AfterSuite(func() {
Expand Down
18 changes: 18 additions & 0 deletions test/tools/imageregistry/catalogd_controller_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
spec:
template:
spec:
containers:
- name: manager
volumeMounts:
- mountPath: /etc/ssl/certs/
name: ca-certs
readOnly: true
volumes:
- name: ca-certs
configMap:
name: docker-registry.operator-controller-e2e.svc
defaultMode: 0644
optional: false
items:
- key: ca-certificates.crt
path: ca-certificates.crt
41 changes: 41 additions & 0 deletions test/tools/imageregistry/imagebuilder.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
apiVersion: batch/v1
kind: Job
metadata:
name: kaniko
namespace: operator-controller-e2e
spec:
template:
spec:
containers:
- name: kaniko
image: gcr.io/kaniko-project/executor:latest
args: ["--dockerfile=/workspace/test-catalog.Dockerfile",
"--context=/workspace/",
"--destination=docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e"]
volumeMounts:
- name: certs
mountPath: /etc/ssl/certs/
readOnly: true
- name: dockerfile
mountPath: /workspace/
- name: build-contents
mountPath: /workspace/test-catalog/
restartPolicy: Never
volumes:
- name: dockerfile
configMap:
name: operator-controller-e2e.dockerfile
items:
- key: test-catalog.Dockerfile
path: test-catalog.Dockerfile
- name: build-contents
configMap:
name: operator-controller-e2e.build-contents
- name: certs
configMap:
name: regcerts
defaultMode: 0644
optional: false
items:
- key: ca-certificates.crt
path: ca-certificates.crt
75 changes: 75 additions & 0 deletions test/tools/imageregistry/imgreg.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
apiVersion: v1
kind: Namespace
metadata:
name: operator-controller-e2e
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: selfsigned-issuer
namespace: operator-controller-e2e
spec:
selfSigned: {}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: operator-controller-e2e-registry
namespace: operator-controller-e2e
spec:
secretName: operator-controller-e2e-registry
isCA: true
dnsNames:
- docker-registry.operator-controller-e2e.svc
privateKey:
algorithm: ECDSA
size: 256
issuerRef:
name: selfsigned-issuer
kind: Issuer
group: cert-manager.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: docker-registry
namespace: operator-controller-e2e
labels:
app: registry
spec:
replicas: 1
selector:
matchLabels:
app: registry
template:
metadata:
labels:
app: registry
spec:
containers:
- name: registry
image: registry:2
volumeMounts:
- name: certs-vol
mountPath: "/certs"
env:
- name: REGISTRY_HTTP_TLS_CERTIFICATE
value: "/certs/tls.crt"
- name: REGISTRY_HTTP_TLS_KEY
value: "/certs/tls.key"
volumes:
- name: certs-vol
secret:
secretName: operator-controller-e2e-registry
---
apiVersion: v1
kind: Service
metadata:
name: docker-registry
namespace: operator-controller-e2e
spec:
selector:
app: registry
ports:
- port: 5000
targetPort: 5000
69 changes: 69 additions & 0 deletions test/tools/imageregistry/registry.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/usr/bin/env bash

set -e

# registry.sh will create an in-cluster image registry useful for end-to-end testing
# of catalogd's unpacking process. It does a few things:
# 1. Installs cert-manager for creating a self-signed certificate for the image registry
# 2. Creates all the resources necessary for deploying the image registry in the operator-controller-e2e namespace
# 3. Creates a ConfigMap containing the CA cert for the image registry to be used by the catalogd-controller-manager
# 4. Creates a ConfigMap containing the CA cert for the image registry to be used by the kaniko pod
# 5. Creates ConfigMaps containing the test catalog + Dockerfile to be mounted to the kaniko pod
# 6. Waits for kaniko pod to have Phase == Succeeded, indicating the test catalog image has been built + pushed
# to the test image registry
# Usage:
# registry.sh

# Install cert-manager
kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/v1.13.1/cert-manager.yaml"
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s

# create the image registry with all the certs
kubectl apply -f test/tools/imageregistry/imgreg.yaml
kubectl wait -n operator-controller-e2e --for=condition=Available deployment/docker-registry --timeout=60s

# get cert value
certval=$(kubectl -n operator-controller-e2e get secret/operator-controller-e2e-registry -o=jsonpath='{.data.ca\.crt}' | base64 -d | sed 's/^/ /')

kubectl apply -f - << EOF
apiVersion: v1
kind: Namespace
metadata:
name: catalogd-system
EOF
# create a ConfigMap that contains the CA certs for the image registry
# This one is created for the catalogd-controller-manager
kubectl apply -f - << EOF
apiVersion: v1
kind: ConfigMap
metadata:
namespace: catalogd-system
name: docker-registry.operator-controller-e2e.svc
data:
"ca-certificates.crt": |
${certval}
EOF

# create a ConfigMap that contains the CA certs for the image registry
# this one is created for the kaniko pod
kubectl apply -f - << EOF
apiVersion: v1
kind: ConfigMap
metadata:
namespace: operator-controller-e2e
name: regcerts
data:
"ca-certificates.crt": |
${certval}
EOF

# Load the testdata onto the cluster as a configmap so it can be used with kaniko
kubectl create configmap -n operator-controller-e2e --from-file=testdata/catalogs/test-catalog.Dockerfile operator-controller-e2e.dockerfile
kubectl create configmap -n operator-controller-e2e --from-file=testdata/catalogs/test-catalog operator-controller-e2e.build-contents

# Create the kaniko pod to build the test image and push it to the test registry.
kubectl apply -f test/tools/imageregistry/imagebuilder.yaml
kubectl wait --for=condition=Complete -n operator-controller-e2e jobs/kaniko --timeout=60s

kubectl patch deployment catalogd-controller-manager -n catalogd-system --patch-file test/tools/imageregistry/catalogd_controller_patch.yaml
kubectl rollout status deployment/catalogd-controller-manager -n catalogd-system --timeout=540s
4 changes: 4 additions & 0 deletions testdata/catalogs/test-catalog.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,2 +1,6 @@
FROM scratch
ADD test-catalog /configs

# Set DC-specific label for the location of the DC root directory
# in the image
LABEL operators.operatorframework.io.index.configs.v1=/configs
2 changes: 2 additions & 0 deletions testdata/catalogs/test-catalog/.indexignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
/expected_all.json
..*

0 comments on commit 6dc5483

Please sign in to comment.