From 976f067236350ab882eba656ead90b71b99f825b Mon Sep 17 00:00:00 2001 From: Kyle Schochenmaier Date: Thu, 30 Jun 2022 16:20:42 -0500 Subject: [PATCH] Add annotations to support specifying userVolumes and userVolumeMounts for the envoy sidecar (#1315) * add envoy user volumes and volumeMounts via annotations --- CHANGELOG.md | 1 + control-plane/connect-inject/annotations.go | 4 + control-plane/connect-inject/envoy_sidecar.go | 11 +++ .../connect-inject/envoy_sidecar_test.go | 63 +++++++++++++ control-plane/connect-inject/mesh_webhook.go | 10 +++ .../connect-inject/mesh_webhook_ent_test.go | 4 +- .../connect-inject/mesh_webhook_test.go | 88 ++++++++++++++++++- 7 files changed, 178 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efd3bb6015..4ed3f02f5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ FEATURES: IMPROVEMENTS: * Control Plane * Added annotations `consul.hashicorp.com/prometheus-ca-file`, `consul.hashicorp.com/prometheus-ca-path`, `consul.hashicorp.com/prometheus-cert-file`, and `consul.hashicorp.com/prometheus-key-file` for configuring TLS scraping on Prometheus metrics endpoints for Envoy sidecars. To enable, set the cert and key file annotations along with one of the ca file/path annotations. [[GH-1303](https://github.com/hashicorp/consul-k8s/pull/1303)] + * Added annotations `consul.hashicorp.com/consul-sidecar-user-volume` and `consul.hashicorp.com/consul-sidecar-user-volume-mount` for attaching Volumes and VolumeMounts to the Envoy sidecar. Both should be JSON objects. [[GH-1315](https://github.com/hashicorp/consul-k8s/pull/1315)] * Helm * Added `connectInject.annotations` and `syncCatalog.annotations` values for setting annotations on connect inject and sync catalog deployments. [[GH-775](https://github.com/hashicorp/consul-k8s/pull/775)] diff --git a/control-plane/connect-inject/annotations.go b/control-plane/connect-inject/annotations.go index 31a4dc7f01..5c8545ee23 100644 --- a/control-plane/connect-inject/annotations.go +++ b/control-plane/connect-inject/annotations.go @@ -86,6 +86,10 @@ const ( annotationConsulSidecarMemoryLimit = "consul.hashicorp.com/consul-sidecar-memory-limit" annotationConsulSidecarMemoryRequest = "consul.hashicorp.com/consul-sidecar-memory-request" + // annotations for sidecar volumes. + annotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume" + annotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount" + // annotations for sidecar concurrency. annotationEnvoyProxyConcurrency = "consul.hashicorp.com/consul-envoy-proxy-concurrency" diff --git a/control-plane/connect-inject/envoy_sidecar.go b/control-plane/connect-inject/envoy_sidecar.go index 77cc7f47bc..c50fbac5ff 100644 --- a/control-plane/connect-inject/envoy_sidecar.go +++ b/control-plane/connect-inject/envoy_sidecar.go @@ -1,6 +1,7 @@ package connectinject import ( + "encoding/json" "fmt" "strconv" "strings" @@ -48,6 +49,16 @@ func (w *MeshWebhook) envoySidecar(namespace corev1.Namespace, pod corev1.Pod, m Command: cmd, } + // Add any extra Envoy VolumeMounts. + if _, ok := pod.Annotations[annotationConsulSidecarUserVolumeMount]; ok { + var volumeMount []corev1.VolumeMount + err := json.Unmarshal([]byte(pod.Annotations[annotationConsulSidecarUserVolumeMount]), &volumeMount) + if err != nil { + return corev1.Container{}, err + } + container.VolumeMounts = append(container.VolumeMounts, volumeMount...) + } + tproxyEnabled, err := transparentProxyEnabled(namespace, pod, w.EnableTransparentProxy) if err != nil { return corev1.Container{}, err diff --git a/control-plane/connect-inject/envoy_sidecar_test.go b/control-plane/connect-inject/envoy_sidecar_test.go index 7727db0fd4..8baa0d30b9 100644 --- a/control-plane/connect-inject/envoy_sidecar_test.go +++ b/control-plane/connect-inject/envoy_sidecar_test.go @@ -394,6 +394,69 @@ func TestHandlerEnvoySidecar_EnvoyExtraArgs(t *testing.T) { } } +func TestHandlerEnvoySidecar_UserVolumeMounts(t *testing.T) { + cases := []struct { + name string + pod corev1.Pod + expectedContainerVolumeMounts []corev1.VolumeMount + expErr string + }{ + { + name: "able to set a sidecar container volume mount via annotation", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotationEnvoyExtraArgs: "--log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + annotationConsulSidecarUserVolumeMount: "[{\"name\": \"tls-cert\", \"mountPath\": \"/custom/path\"}, {\"name\": \"tls-ca\", \"mountPath\": \"/custom/path2\"}]", + }, + }, + }, + expectedContainerVolumeMounts: []corev1.VolumeMount{ + { + Name: "consul-connect-inject-data", + MountPath: "/consul/connect-inject", + }, + { + Name: "tls-cert", + MountPath: "/custom/path", + }, + { + Name: "tls-ca", + MountPath: "/custom/path2", + }, + }, + }, + { + name: "invalid annotation results in error", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotationEnvoyExtraArgs: "--log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + annotationConsulSidecarUserVolumeMount: "[abcdefg]", + }, + }, + }, + expErr: "invalid character 'a' looking ", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := MeshWebhook{ + ImageConsul: "hashicorp/consul:latest", + ImageEnvoy: "hashicorp/consul-k8s:latest", + } + c, err := h.envoySidecar(testNS, tc.pod, multiPortInfo{}) + if tc.expErr == "" { + require.NoError(t, err) + require.Equal(t, tc.expectedContainerVolumeMounts, c.VolumeMounts) + } else { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expErr) + } + }) + } +} + func TestHandlerEnvoySidecar_Resources(t *testing.T) { mem1 := resource.MustParse("100Mi") mem2 := resource.MustParse("200Mi") diff --git a/control-plane/connect-inject/mesh_webhook.go b/control-plane/connect-inject/mesh_webhook.go index 73b56704fb..cd1e49cd25 100644 --- a/control-plane/connect-inject/mesh_webhook.go +++ b/control-plane/connect-inject/mesh_webhook.go @@ -220,6 +220,16 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi // Optionally mount data volume to other containers w.injectVolumeMount(pod) + // Optionally add any volumes that are to be used by the envoy sidecar. + if _, ok := pod.Annotations[annotationConsulSidecarUserVolume]; ok { + var userVolumes []corev1.Volume + err := json.Unmarshal([]byte(pod.Annotations[annotationConsulSidecarUserVolume]), &userVolumes) + if err != nil { + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error unmarshalling sidecar user volumes: %s", err)) + } + pod.Spec.Volumes = append(pod.Spec.Volumes, userVolumes...) + } + // Add the upstream services as environment variables for easy // service discovery. containerEnvVars := w.containerEnvVars(pod) diff --git a/control-plane/connect-inject/mesh_webhook_ent_test.go b/control-plane/connect-inject/mesh_webhook_ent_test.go index 5faaf4e0c1..7a34ee3d73 100644 --- a/control-plane/connect-inject/mesh_webhook_ent_test.go +++ b/control-plane/connect-inject/mesh_webhook_ent_test.go @@ -30,7 +30,7 @@ func TestHandler_MutateWithNamespaces(t *testing.T) { basicSpec := corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "web", }, }, @@ -285,7 +285,7 @@ func TestHandler_MutateWithNamespaces(t *testing.T) { func TestHandler_MutateWithNamespaces_ACLs(t *testing.T) { basicSpec := corev1.PodSpec{ Containers: []corev1.Container{ - corev1.Container{ + { Name: "web", }, }, diff --git a/control-plane/connect-inject/mesh_webhook_test.go b/control-plane/connect-inject/mesh_webhook_test.go index bbebd48c30..8b37462506 100644 --- a/control-plane/connect-inject/mesh_webhook_test.go +++ b/control-plane/connect-inject/mesh_webhook_test.go @@ -384,7 +384,93 @@ func TestHandlerHandle(t *testing.T) { }, }, }, - + { + "pod with sidecar volume mount annotation", + MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotationConsulSidecarUserVolume: "[{\"name\":\"bbb\",\"csi\":{\"driver\":\"bob\"}}]", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(keyInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(annotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + }, + }, + { + "pod with sidecar invalid volume mount annotation", + MeshWebhook{ + Log: logrtest.TestLogger{T: t}, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + annotationConsulSidecarUserVolume: "[a]", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + }), + }, + }, + "error unmarshalling sidecar user volumes: invalid character 'a' looking for beginning of value", + nil, + }, { "pod with service annotation", MeshWebhook{