test: add encryptionAtRest config in capi-quick-start (#659)

**What problem does this PR solve?**:
This PR is stacked on
#610
- adds encryptionAtRest variable to all capi-quick-start examples
- documentation for encryptionAtRest

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
Caren e2e tests
Manually on docker and aws

docs/content/customization/generic/encryption-at-rest.md

@@ -0,0 +1,66 @@
++++
+title = "Encryption At Rest"
++++
+
+`encryptionAtRest` variable enables encrypting kubernetes resources at rest using provided encryption provider.
+When this variable is set, kuberntetes `secrets` and `configmap`s are encrypted before writing them at `etcd`.
+
+If the `encryptionAtRest` property is not specified, then
+the customization will be skipped. The `secrets` and `configmaps` will not be stored as encrypted in `etcd`.
+
+We support following encryption providers
+
+- aescbc
+- secretbox
+
+More information about encryption at-rest: [Encrypting Confidential Data at Rest
+](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/)
+
+## Example
+
+To encrypt `configmaps` and `secrets` kubernetes resources using `aescbc` encryption provider:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          encryptionAtRest:
+            providers:
+              - aescbc: {}
+```
+
+Applying this configuration will result in
+
+1. `<CLUSTER_NAME>-encryption-config` secret generated.
+
+  A secret key for the encryption provider is generated and stored in `<CLUSTER_NAME>-encryption-config` secret.
+  The APIServer will be configured to use the secret key to encrypt `secrets` and
+   `configmaps` kubernetes resources before writing them to etcd.
+  When reading resources from `etcd`, encryption provider that matches the stored data attempts in order to decrypt the data.
+  CAREN currently does not rotate the key once it generated.
+
+1. Configure APIServer with encryption configuration:
+
+- `KubeadmControlPlaneTemplate`:
+
+  ```yaml
+    spec:
+      kubeadmConfigSpec:
+        clusterConfiguration:
+          apiServer:
+            extraArgs:
+              encryption-provider-config: /etc/kubernetes/pki/encryptionconfig.yaml
+      files:
+        - contentFrom:
+            secret:
+              key: config
+              name: <CLUSTER_NAME>-encryption-config
+          path: /etc/kubernetes/pki/encryptionconfig.yaml
+          permissions: "0640"
+  ```

examples/capi-quick-start/aws-cluster-calico-crs.yaml

@@ -47,6 +47,9 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value:
         aws:

examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml

@@ -47,6 +47,9 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value:
         aws:

examples/capi-quick-start/aws-cluster-cilium-crs.yaml

@@ -47,6 +47,9 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value:
         aws:

examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml

@@ -47,6 +47,9 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value:
         aws:

examples/capi-quick-start/docker-cluster-calico-crs.yaml

@@ -29,6 +29,9 @@ spec:
             strategy: ClusterResourceSet
           nfd:
             strategy: ClusterResourceSet
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value: {}
     version: ${KUBERNETES_VERSION}

examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml

@@ -29,6 +29,9 @@ spec:
             strategy: HelmAddon
           nfd:
             strategy: HelmAddon
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value: {}
     version: ${KUBERNETES_VERSION}

examples/capi-quick-start/docker-cluster-cilium-crs.yaml

@@ -29,6 +29,9 @@ spec:
             strategy: ClusterResourceSet
           nfd:
             strategy: ClusterResourceSet
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value: {}
     version: ${KUBERNETES_VERSION}

examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml

@@ -29,6 +29,9 @@ spec:
             strategy: HelmAddon
           nfd:
             strategy: HelmAddon
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
     - name: workerConfig
       value: {}
     version: ${KUBERNETES_VERSION}

examples/capi-quick-start/nutanix-cluster-calico-crs.yaml

@@ -105,6 +105,9 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
         imageRegistries:
         - credentials:
             secretRef:

examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml

@@ -105,6 +105,9 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
         imageRegistries:
         - credentials:
             secretRef:

examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml

@@ -105,6 +105,9 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
         imageRegistries:
         - credentials:
             secretRef:

examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml

@@ -105,6 +105,9 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        encryptionAtRest:
+          providers:
+          - aescbc: {}
         imageRegistries:
         - credentials:
             secretRef:

hack/examples/bases/aws/cluster/kustomization.yaml.tmpl

@@ -45,6 +45,9 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/aws/config-var.yaml
+- target:
+    kind: Cluster
+  path: ../../../patches/encryption.yaml
 
 # Delete the clusterclass-specific resources.
 - target:

hack/examples/bases/docker/cluster/kustomization.yaml.tmpl

@@ -35,3 +35,6 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/cluster-autoscaler.yaml
+- target:
+    kind: Cluster
+  path: ../../../patches/encryption.yaml

hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl

@@ -36,6 +36,9 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/nutanix/initialize-variables.yaml
+- target:
+    kind: Cluster
+  path: ../../../patches/encryption.yaml
 
 # Remove Additional Trust Bundle ConfigMap
 - target:

hack/examples/patches/encryption.yaml

@@ -0,0 +1,8 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+- op: "add"
+  path: "/spec/topology/variables/0/value/encryptionAtRest"
+  value:
+    providers:
+      - aescbc: {}

test/e2e/ownerreference_helpers.go

@@ -167,8 +167,8 @@ var (
 	// https://github.com/kubernetes-sigs/cluster-api/tree/main/docs/book/src/reference/owner_references.md.
 	KubernetesReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
 		secretKind: func(owners []metav1.OwnerReference) error {
-			// TODO:deepakm-ntnx Currently pc-creds, pc-creds-for-csi, dockerhub-credentials
-			// and registry-creds have unexpected owners which needs more investigation
+			// TODO:deepakm-ntnx Currently pc-creds, pc-creds-for-csi, dockerhub-credentials,
+			// registry-creds, and encryption config secrets have unexpected owners which needs more investigation.
 			return nil
 		},
 		configMapKind: func(owners []metav1.OwnerReference) error {

test/e2e/quick_start_test.go

@@ -93,7 +93,7 @@ var _ = Describe("Quick start", Serial, func() {
 										framework.DockerInfraOwnerReferenceAssertions,
 										framework.KubeadmBootstrapOwnerReferenceAssertions,
 										framework.KubeadmControlPlaneOwnerReferenceAssertions,
-										framework.KubernetesReferenceAssertions,
+										AWSInfraOwnerReferenceAssertions,
 										NutanixInfraOwnerReferenceAssertions,
 										AddonReferenceAssertions,
 										KubernetesReferenceAssertions,