Add webhook certificate handling for k8s

Adds support for webhooks in k8s deployments through options to handle webhoook certificate configuration, either via providing the CA bundle through environment variable or cert-manager annotations.
k8snetworkplumbingwg · Apr 14, 2021 · ca5ee80 · ca5ee80
1 parent bbc0397
commit ca5ee80
Show file tree

Hide file tree

Showing 9 changed files with 112 additions and 4 deletions.
diff --git a/Makefile b/Makefile
@@ -18,7 +18,6 @@ IMAGE_TAG?=nfvpe/$(APP_NAME):latest
 MAIN_PKG=cmd/manager/main.go
 export NAMESPACE?=openshift-sriov-network-operator
 export WATCH_NAMESPACE?=openshift-sriov-network-operator
-export ENABLE_ADMISSION_CONTROLLER?=true
 export GOFLAGS+=-mod=vendor
 export GO111MODULE=on
 PKGS=$(shell go list ./... | grep -v -E '/vendor/|/test|/examples')
@@ -189,11 +188,12 @@ bundle: manifests
 bundle-build:
 	docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) .
 
+deploy-setup: export ENABLE_ADMISSION_CONTROLLER?=true
 deploy-setup: skopeo install
 	hack/deploy-setup.sh $(NAMESPACE)
 
 deploy-setup-k8s: export NAMESPACE=sriov-network-operator
-deploy-setup-k8s: export ENABLE_ADMISSION_CONTROLLER=false
+deploy-setup-k8s: export ENABLE_ADMISSION_CONTROLLER?=false
 deploy-setup-k8s: export CNI_BIN_PATH=/opt/cni/bin
 deploy-setup-k8s: export OPERATOR_EXEC=kubectl
 deploy-setup-k8s: export CLUSTER_TYPE=kubernetes

diff --git a/bindata/manifests/operator-webhook/001-service.yaml b/bindata/manifests/operator-webhook/001-service.yaml
@@ -5,7 +5,9 @@ metadata:
   name: operator-webhook-service
   namespace: {{.Namespace}}
   annotations:
+    {{if and (not .CaBundle) (eq .ClusterType "openshift")}}
     service.alpha.openshift.io/serving-cert-secret-name: operator-webhook-service
+    {{end}}
 spec:
   ports:
   - port: 443

diff --git a/bindata/manifests/operator-webhook/003-webhook.yaml b/bindata/manifests/operator-webhook/003-webhook.yaml
@@ -5,7 +5,11 @@ metadata:
   name: {{.SRIOVMutatingWebhookName}}
   namespace: {{.Namespace}}
   annotations:
+    {{if and (not .CaBundle) (eq .ClusterType "openshift") }}
     service.beta.openshift.io/inject-cabundle: "true"
+    {{else if and (not .CaBundle) (eq .ClusterType "kubernetes") }}
+    cert-manager.io/inject-ca-from: {{.Namespace}}/operator-webhook-service
+    {{end}}
 webhooks:
   - name: operator-webhook.sriovnetwork.openshift.io
     failurePolicy: Fail
@@ -14,6 +18,9 @@ webhooks:
         name: operator-webhook-service
         namespace: {{.Namespace}}
         path: "/mutating-custom-resource"
+      {{if .CaBundle}}
+      caBundle: "{{.CaBundle}}"
+      {{end}}
     rules:
       - operations: [ "CREATE", "UPDATE" ]
         apiGroups: ["sriovnetwork.openshift.io"]
@@ -27,7 +34,11 @@ metadata:
   name: {{.SRIOVMutatingWebhookName}}
   namespace: {{.Namespace}}
   annotations:
+    {{if and (not .CaBundle) (eq .ClusterType "openshift")}}
     service.beta.openshift.io/inject-cabundle: "true"
+    {{else if and (not .CaBundle) (eq .ClusterType "kubernetes")}}
+    cert-manager.io/inject-ca-from: {{.Namespace}}/operator-webhook-service
+    {{end}}
 webhooks:
   - name: operator-webhook.sriovnetwork.openshift.io
     failurePolicy: Fail
@@ -36,6 +47,9 @@ webhooks:
         name: operator-webhook-service
         namespace: {{.Namespace}}
         path: "/validating-custom-resource"
+      {{if .CaBundle}}
+      caBundle: "{{.CaBundle}}"
+      {{end}}
     rules:
       - operations: [ "CREATE", "UPDATE", "DELETE" ]
         apiGroups: ["sriovnetwork.openshift.io"]

diff --git a/bindata/manifests/webhook/001-service.yaml b/bindata/manifests/webhook/001-service.yaml
@@ -5,7 +5,9 @@ metadata:
   name: network-resources-injector-service
   namespace: {{.Namespace}}
   annotations:
+    {{if and (not .CaBundle) (eq .ClusterType "openshift")}}
     service.alpha.openshift.io/serving-cert-secret-name: network-resources-injector-secret
+    {{end}}
 spec:
   ports:
   - port: 443

diff --git a/bindata/manifests/webhook/003-webhook.yaml b/bindata/manifests/webhook/003-webhook.yaml
@@ -5,14 +5,21 @@ metadata:
   name: {{.SRIOVMutatingWebhookName}}
   namespace: {{.Namespace}}
   annotations:
+    {{if and (not .CaBundle) (eq .ClusterType "openshift")}}
     service.beta.openshift.io/inject-cabundle: "true"
+    {{else if and (not .CaBundle) (eq .ClusterType "kubernetes")}}
+    cert-manager.io/inject-ca-from: {{.Namespace}}/network-resources-injector-secret
+    {{end}}
 webhooks:
   - name: network-resources-injector-config.k8s.io
     clientConfig:
       service:
         name: network-resources-injector-service
         namespace: {{.Namespace}}
         path: "/mutate"
+      {{if .CaBundle}}
+      caBundle: "{{.CaBundle}}"
+      {{end}}
     rules:
       - operations: [ "CREATE" ]
         apiGroups: ["apps", ""]

diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go
@@ -213,6 +213,8 @@ func (r *SriovOperatorConfigReconciler) syncWebhookObjs(dc *sriovnetworkv1.Sriov
 		data.Data["NetworkResourcesInjectorImage"] = os.Getenv("NETWORK_RESOURCES_INJECTOR_IMAGE")
 		data.Data["SriovNetworkWebhookImage"] = os.Getenv("SRIOV_NETWORK_WEBHOOK_IMAGE")
 		data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
+		data.Data["ClusterType"] = utils.ClusterType
+		data.Data["CaBundle"] = os.Getenv("WEBHOOK_CA_BUNDLE")
 		objs, err := render.RenderDir(path, &data)
 		if err != nil {
 			logger.Error(err, "Fail to render webhook manifests")

diff --git a/deploy/operator.yaml b/deploy/operator.yaml
@@ -63,3 +63,5 @@ spec:
               value: $CNI_BIN_PATH
             - name: CLUSTER_TYPE
               value: $CLUSTER_TYPE
+            - name: WEBHOOK_CA_BUNDLE
+              value: "$WEBHOOK_CA_BUNDLE"
diff --git a/doc/quickstart.md b/doc/quickstart.md
@@ -54,11 +54,89 @@ make deploy-setup
 
 If you are running a Kubernetes cluster:
 ```bash
-export OPERATOR_EXEC=kubectl
-export ENABLE_ADMISSION_CONTROLLER=false
 make deploy-setup-k8s
 ```
 
+Webhooks are disabled when deploying on a Kubernetes cluster as per the instructions above. To enable webhooks on Kubernetes cluster, there are two options:
+
+1. Create certificates for each of the two webhooks using a single CA whose cert you provide through an environment variable.
+
+   For example, given `cacert.pem`, `key.pem` and `cert.pem`:
+   ```bash
+   kubectl -n sriov-network-operator create secret tls operator-webhook-service --cert=cert.pem --key=key.pem
+   kubectl -n sriov-network-operator create secret tls network-resources-injector-secret --cert=cert.pem --key=key.pem
+   export ENABLE_ADMISSION_CONTROLLER=true
+   export WEBHOOK_CA_BUNDLE=$(base64 < cacert.pem | tr -d '\n') 
+   make deploy-setup-k8s
+   ```
+
+2. Using https://cert-manager.io/, deploy it as:
+   ```bash
+   kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.0/cert-manager.yaml
+   ```
+
+   Define the appropriate Issuer and Certificates, as an example:
+   ```yaml
+   apiVersion: cert-manager.io/v1
+   kind: ClusterIssuer
+   metadata:
+     name: selfsigned-issuer
+   spec:
+     selfSigned: {}
+   ---
+   apiVersion: cert-manager.io/v1
+   kind: Certificate
+   metadata:
+     name: sriov-network-operator-ca
+     namespace: sriov-network-operator
+   spec:
+     isCA: true
+     commonName: sriov-network-operator-ca
+     secretName: sriov-network-operator-ca
+     issuerRef:
+       name: selfsigned-issuer
+       kind: ClusterIssuer
+   ---
+   apiVersion: cert-manager.io/v1
+   kind: Issuer
+   metadata:
+     name: sriov-network-operator-ca
+     namespace: sriov-network-operator
+   spec:
+     ca:
+       secretName: sriov-network-operator-ca
+   ---
+   apiVersion: cert-manager.io/v1
+   kind: Certificate
+   metadata:
+     name: operator-webhook-service
+     namespace: sriov-network-operator
+   spec:
+     secretName: operator-webhook-service
+     dnsNames:
+     - operator-webhook-service.sriov-network-operator.svc
+     issuerRef:
+       name: sriov-network-operator-ca
+   ---
+   apiVersion: cert-manager.io/v1
+   kind: Certificate
+   metadata:
+     name: network-resources-injector-service
+     namespace: sriov-network-operator
+   spec:
+     secretName: network-resources-injector-secret
+     dnsNames:
+     - network-resources-injector-service.sriov-network-operator.svc
+     issuerRef:
+       name: sriov-network-operator-ca
+   ```
+
+    And then deploy the operator:
+    ```bash
+    export ENABLE_ADMISSION_CONTROLLER=true
+    make deploy-setup-k8s
+    ```
+
 By default, the operator will be deployed in namespace 'sriov-network-operator' for Kubernetes cluster, you can check if the deployment is finished successfully.
 
 ```bash

diff --git a/hack/env.sh b/hack/env.sh
@@ -34,3 +34,4 @@ export RESOURCE_PREFIX=${RESOURCE_PREFIX:-openshift.io}
 export ENABLE_ADMISSION_CONTROLLER=${ENABLE_ADMISSION_CONTROLLER:-"true"}
 export CLUSTER_TYPE=${CLUSTER_TYPE:-openshift}
 export NAMESPACE=${NAMESPACE:-"openshift-sriov-network-operator"}
+export WEBHOOK_CA_BUNDLE=${WEBHOOK_CA_BUNDLE:-""}