From 73b1eb7736cdd47505d05b46725cac8e5d6c1ca7 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Fri, 17 May 2024 01:45:11 -0700 Subject: [PATCH] test: add encryptionAtRest config in capi-quick-start (#659) **What problem does this PR solve?**: This PR is stacked on https://github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pull/610 - adds encryptionAtRest variable to all capi-quick-start examples - documentation for encryptionAtRest **How Has This Been Tested?**: Caren e2e tests Manually on docker and aws --- .../generic/encryption-at-rest.md | 66 +++++++++++++++++++ .../aws-cluster-calico-crs.yaml | 3 + .../aws-cluster-calico-helm-addon.yaml | 3 + .../aws-cluster-cilium-crs.yaml | 3 + .../aws-cluster-cilium-helm-addon.yaml | 3 + .../docker-cluster-calico-crs.yaml | 3 + .../docker-cluster-calico-helm-addon.yaml | 3 + .../docker-cluster-cilium-crs.yaml | 3 + .../docker-cluster-cilium-helm-addon.yaml | 3 + .../nutanix-cluster-calico-crs.yaml | 3 + .../nutanix-cluster-calico-helm-addon.yaml | 3 + .../nutanix-cluster-cilium-crs.yaml | 3 + .../nutanix-cluster-cilium-helm-addon.yaml | 3 + .../bases/aws/cluster/kustomization.yaml.tmpl | 3 + .../docker/cluster/kustomization.yaml.tmpl | 3 + .../nutanix/cluster/kustomization.yaml.tmpl | 3 + hack/examples/patches/encryption.yaml | 8 +++ test/e2e/ownerreference_helpers.go | 4 +- test/e2e/quick_start_test.go | 2 +- 19 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 docs/content/customization/generic/encryption-at-rest.md create mode 100644 hack/examples/patches/encryption.yaml diff --git a/docs/content/customization/generic/encryption-at-rest.md b/docs/content/customization/generic/encryption-at-rest.md new file mode 100644 index 000000000..71f6a47bf --- /dev/null +++ b/docs/content/customization/generic/encryption-at-rest.md @@ -0,0 +1,66 @@ ++++ +title = "Encryption At Rest" ++++ + +`encryptionAtRest` variable enables encrypting kubernetes resources at rest using provided encryption provider. +When this variable is set, kuberntetes `secrets` and `configmap`s are encrypted before writing them at `etcd`. + +If the `encryptionAtRest` property is not specified, then +the customization will be skipped. The `secrets` and `configmaps` will not be stored as encrypted in `etcd`. + +We support following encryption providers + +- aescbc +- secretbox + +More information about encryption at-rest: [Encrypting Confidential Data at Rest +](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) + +## Example + +To encrypt `configmaps` and `secrets` kubernetes resources using `aescbc` encryption provider: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + encryptionAtRest: + providers: + - aescbc: {} +``` + +Applying this configuration will result in + +1. `-encryption-config` secret generated. + + A secret key for the encryption provider is generated and stored in `-encryption-config` secret. + The APIServer will be configured to use the secret key to encrypt `secrets` and + `configmaps` kubernetes resources before writing them to etcd. + When reading resources from `etcd`, encryption provider that matches the stored data attempts in order to decrypt the data. + CAREN currently does not rotate the key once it generated. + +1. Configure APIServer with encryption configuration: + +- `KubeadmControlPlaneTemplate`: + + ```yaml + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + encryption-provider-config: /etc/kubernetes/pki/encryptionconfig.yaml + files: + - contentFrom: + secret: + key: config + name: -encryption-config + path: /etc/kubernetes/pki/encryptionconfig.yaml + permissions: "0640" + ``` diff --git a/examples/capi-quick-start/aws-cluster-calico-crs.yaml b/examples/capi-quick-start/aws-cluster-calico-crs.yaml index 790b4a640..f67b253fb 100644 --- a/examples/capi-quick-start/aws-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-crs.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index 52bf9648a..7d06d032c 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml index e9541e789..c0f4acff1 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index e803994e8..fb6441e0f 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -47,6 +47,9 @@ spec: baseOS: ${AMI_LOOKUP_BASEOS} format: ${AMI_LOOKUP_FORMAT} org: "${AMI_LOOKUP_ORG}" + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: aws: diff --git a/examples/capi-quick-start/docker-cluster-calico-crs.yaml b/examples/capi-quick-start/docker-cluster-calico-crs.yaml index 69673293e..551aab200 100644 --- a/examples/capi-quick-start/docker-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-crs.yaml @@ -29,6 +29,9 @@ spec: strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml index 459f248a0..53c5cc4e7 100644 --- a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml @@ -29,6 +29,9 @@ spec: strategy: HelmAddon nfd: strategy: HelmAddon + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml index 7409544d2..0688f562a 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml @@ -29,6 +29,9 @@ spec: strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml index 8c2b7ef5e..1eda93a6e 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml @@ -29,6 +29,9 @@ spec: strategy: HelmAddon nfd: strategy: HelmAddon + encryptionAtRest: + providers: + - aescbc: {} - name: workerConfig value: {} version: ${KUBERNETES_VERSION} diff --git a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml index 1680514c4..6d30f72db 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml index 3a68dfa6f..cad76de30 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml index f5e464ddb..fc8d480bc 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml index 5c1d2ce3b..0dd369e6c 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml @@ -105,6 +105,9 @@ spec: systemDiskSize: 40Gi vcpuSockets: 2 vcpusPerSocket: 1 + encryptionAtRest: + providers: + - aescbc: {} imageRegistries: - credentials: secretRef: diff --git a/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl b/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl index b90db5a0b..0e3f8436c 100644 --- a/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/aws/cluster/kustomization.yaml.tmpl @@ -45,6 +45,9 @@ patches: - target: kind: Cluster path: ../../../patches/aws/config-var.yaml +- target: + kind: Cluster + path: ../../../patches/encryption.yaml # Delete the clusterclass-specific resources. - target: diff --git a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl index 22bfa1a47..7a4e271eb 100644 --- a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl @@ -35,3 +35,6 @@ patches: - target: kind: Cluster path: ../../../patches/cluster-autoscaler.yaml +- target: + kind: Cluster + path: ../../../patches/encryption.yaml diff --git a/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl b/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl index 843bf5281..b03bee535 100644 --- a/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl @@ -36,6 +36,9 @@ patches: - target: kind: Cluster path: ../../../patches/nutanix/initialize-variables.yaml +- target: + kind: Cluster + path: ../../../patches/encryption.yaml # Remove Additional Trust Bundle ConfigMap - target: diff --git a/hack/examples/patches/encryption.yaml b/hack/examples/patches/encryption.yaml new file mode 100644 index 000000000..b96c73a89 --- /dev/null +++ b/hack/examples/patches/encryption.yaml @@ -0,0 +1,8 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/encryptionAtRest" + value: + providers: + - aescbc: {} diff --git a/test/e2e/ownerreference_helpers.go b/test/e2e/ownerreference_helpers.go index 9e67d6208..b6915ac2a 100644 --- a/test/e2e/ownerreference_helpers.go +++ b/test/e2e/ownerreference_helpers.go @@ -167,8 +167,8 @@ var ( // https://github.com/kubernetes-sigs/cluster-api/tree/main/docs/book/src/reference/owner_references.md. KubernetesReferenceAssertions = map[string]func([]metav1.OwnerReference) error{ secretKind: func(owners []metav1.OwnerReference) error { - // TODO:deepakm-ntnx Currently pc-creds, pc-creds-for-csi, dockerhub-credentials - // and registry-creds have unexpected owners which needs more investigation + // TODO:deepakm-ntnx Currently pc-creds, pc-creds-for-csi, dockerhub-credentials, + // registry-creds, and encryption config secrets have unexpected owners which needs more investigation. return nil }, configMapKind: func(owners []metav1.OwnerReference) error { diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index 4dc061ca3..8f8bd0564 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -93,7 +93,7 @@ var _ = Describe("Quick start", Serial, func() { framework.DockerInfraOwnerReferenceAssertions, framework.KubeadmBootstrapOwnerReferenceAssertions, framework.KubeadmControlPlaneOwnerReferenceAssertions, - framework.KubernetesReferenceAssertions, + AWSInfraOwnerReferenceAssertions, NutanixInfraOwnerReferenceAssertions, AddonReferenceAssertions, KubernetesReferenceAssertions,