Etcd encryption feature refactor for deployment and upgrades

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/copy-kubernetes-pki.yml

@@ -23,7 +23,7 @@
       pki_front_proxy_ca_key: pki/front-proxy-ca.key
     _optional_pki_files_map:
       pki_etcd_etc_encryption_conf: >-
-        {{ 'pki/etcd/etc-encryption.conf' if specification.advanced.etcd_args.encrypted else '' }}
+        {{ 'pki/etcd/etc-encryption.conf' if (specification.advanced.etcd_args.encrypted | bool) else '' }}
 
 - name: Check if the PKI file exists
   delegate_to: localhost

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/etcd-encryption-init.yml

@@ -0,0 +1,18 @@
+---
+- name: Ensure the /etc/kubernetes/pki/etcd/ directory exists
+  file:
+    path: /etc/kubernetes/pki/etcd/
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=
+
+# Please note, the config file is automatically redistributed to other masters (via copy-kubernetes-pki.yml).
+- name: Render the etcd encryption config file (just one time)
+  template:
+    dest: /etc/kubernetes/pki/etcd/etc-encryption.conf
+    src: etc-encryption.conf.j2
+    force: false  # please keep it "false" here, otherwise the secret will change and your cluster will start failing eventually
+  vars:
+    etcd_encryption_secret: >-
+      {{ lookup('password', '/dev/null length=32') | b64encode }}

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/main.yml

@@ -9,9 +9,9 @@
 - name: Init Kubernetes master
   when: kubernetes_common.automation_designated_master == inventory_hostname
   block:
+    - import_tasks: etcd-encryption-init.yml
+      when: specification.advanced.etcd_args.encrypted | bool
     - import_tasks: master-init.yml
-    - import_tasks: etcd-encryption.yml
-      when: specification.advanced.etcd_args.encrypted
     - import_tasks: registry-secrets.yml
     - import_tasks: copy-kubernetes-pki.yml
 
@@ -22,8 +22,6 @@
       block:
         - import_tasks: copy-kubernetes-pki.yml
         - import_tasks: master-join.yml
-        - import_tasks: etcd-encryption.yml
-          when: specification.advanced.etcd_args.encrypted
 
 - import_tasks: master-untaint.yml
 

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/etc-encryption.conf.j2

@@ -7,5 +7,5 @@ resources:
     - aescbc:
         keys:
         - name: key1
-          secret: "{{ random_secret.stdout }}"
-    - identity: {}
+          secret: "{{ etcd_encryption_secret }}"
+    - identity: {}

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2

@@ -10,6 +10,9 @@ controlPlaneEndpoint: "localhost:3446"
 apiServer:
   timeoutForControlPlane: 4m0s
   extraArgs: # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
+{% if specification.advanced.etcd_args.encrypted | bool %}
+    encryption-provider-config: /etc/kubernetes/pki/etcd/etc-encryption.conf
+{% endif %}
 {% for key, value in specification.advanced.api_server_args.items() %}
     {{ key }}: "{{ value }}"
 {% endfor %}

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/patch-kubeadm-config.yml

@@ -0,0 +1,132 @@
+---
+- name: Check the etc-encryption.conf file
+  stat:
+    path: &etc-encryption-conf /etc/kubernetes/pki/etcd/etc-encryption.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: stat_etcd_encryption_config_file
+
+# Assuming that if the etcd encryption config file is absent, then
+# the encryption feature has never been enabled for the cluster at hand.
+- when:
+    - stat_etcd_encryption_config_file.stat.exists
+  block:
+    - name: Check the kubeadm-config.yml file
+      stat:
+        path: &kubeadm-config-yml /etc/kubeadm/kubeadm-config.yml
+        get_attributes: false
+        get_checksum: false
+        get_mime: false
+      register: stat_kubeadm_config_file
+
+- when:
+    - stat_etcd_encryption_config_file.stat.exists
+    - stat_kubeadm_config_file.stat.exists
+  block:
+    - name: Load contents of the kubeadm-config.yml file
+      slurp:
+        path: *kubeadm-config-yml
+      register: slurp_kubeadm_config
+
+    - name: Save modified contents of the kubeadm-config.yml file
+      copy:
+        dest: *kubeadm-config-yml
+
+        # Save all documents.
+        content: |
+          {% for document in _documents_updated %}
+          ---
+          {{ document | to_nice_yaml(indent=2) }}
+          {% endfor -%}
+
+      vars:
+        # Parse yaml payload (remove empty documents).
+        _documents: >-
+          {{ slurp_kubeadm_config.content | b64decode
+                                          | from_yaml_all
+                                          | select
+                                          | list }}
+        # Prepare the patch.
+        # In this patch we include location of the etcd encryption config file.
+        # If it is not included, then the etcd encryption feature becomes disabled/broken.
+        # If it is not present on a cluster that has kube-system secrets already encrypted, then
+        # it may cause any upgrade attempt to freeze for a very long time (in Epiphany it has been reported to be even up to 8 hours).
+        _update:
+          apiServer:
+            extraArgs:
+              encryption-provider-config: *etc-encryption-conf
+
+        # Process all documents (returns a list of dictionaries).
+        _documents_updated: >-
+          {%- set output = [] -%}
+          {%- for document in _documents -%}
+            {%- if document.kind is defined and document.kind == 'ClusterConfiguration' -%}
+              {{- output.append(document | combine(_update, recursive=true)) -}}
+            {%- else -%}
+              {{- output.append(document) -}}
+            {%- endif -%}
+          {%- endfor -%}
+          {{- output -}}
+
+# The `kubeadm upgrade` command can be executed with or without a config file.
+# If the kubeadm-config.yml file does not exists, then we at least patch the kubeadm-config configmap.
+- when:
+    - stat_etcd_encryption_config_file.stat.exists
+    - not stat_kubeadm_config_file.stat.exists
+  run_once: true  # makes no sense to execute it more than once (would be redundant)
+  block:
+    - name: Load the kubeadm-config configmap
+      shell: |
+        kubectl get configmap kubeadm-config \
+          --namespace kube-system \
+          --output yaml
+      args:
+        executable: /bin/bash
+      environment:
+        KUBECONFIG: &KUBECONFIG /etc/kubernetes/admin.conf
+      register: shell_kubeadm_configmap
+      changed_when: false
+
+    # The following procedure ensures that etcd encryption is always enabled
+    # during subsequent kubeadm executions (if the config file is not present).
+    - name: Patch and re-apply the kubeadm-config configmap
+      shell: |
+        kubectl apply \
+          --namespace kube-system \
+          --filename - \
+          <<< "$KUBEADM_CONFIGMAP_DOCUMENT"
+      args:
+        executable: /bin/bash
+      environment:
+        KUBECONFIG: *KUBECONFIG
+        # Render an altered kubeadm-config configmap document.
+        KUBEADM_CONFIGMAP_DOCUMENT: >-
+          {{ _document | combine(_update2, recursive=true) | to_nice_yaml(indent=2) }}
+
+      # Skip the task if there is no change in the cluster config.
+      when: _cluster_config_updated != _cluster_config  # comparing two dictionaries here
+
+      vars:
+        # Parse yaml payload.
+        _document: >-
+          {{ shell_kubeadm_configmap.stdout | from_yaml }}
+
+        # Extract cluster config.
+        _cluster_config: >-
+          {{ _document.data.ClusterConfiguration | from_yaml }}
+
+        # Prepare the cluster config patch.
+        _update1:
+          apiServer:
+            extraArgs:
+              encryption-provider-config: *etc-encryption-conf
+
+        _cluster_config_updated: >-
+          {{ _cluster_config | combine(_update1, recursive=true) }}
+
+        # Prepare the final update for the whole document.
+        _update2:
+          data:
+            ClusterConfiguration: >-
+              {{ _cluster_config_updated | to_nice_yaml(indent=2) }}

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kubernetes/upgrade-kubeadm-config.yml

@@ -1,7 +1,58 @@
+---
+- name: Run assertions for parameters of the task file at hand
+  block:
+    - assert:
+        that:
+          - version is defined
+          - version is string
+          - version | length > 0
+        fail_msg: "Invalid version string."
 
-- name: Update kubeadm-config.yml with current version (v{{ version }})
-  lineinfile:
-    dest: /etc/kubeadm/kubeadm-config.yml
-    regexp: "^kubernetesVersion:"
-    line: "kubernetesVersion: v{{ version }}"
-    state: present
+- name: Check the kubeadm-config.yml file
+  stat:
+    path: &kubeadm-config-yml /etc/kubeadm/kubeadm-config.yml
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: stat_kubeadm_config_file
+
+- when: stat_kubeadm_config_file.stat.exists
+  block:
+    - name: Load contents of the kubeadm-config.yml file
+      slurp:
+        path: *kubeadm-config-yml
+      register: slurp_kubeadm_config
+
+    - name: Save modified contents of the kubeadm-config.yml file
+      copy:
+        dest: *kubeadm-config-yml
+
+        # Save all documents.
+        content: |
+          {% for document in _documents_updated %}
+          ---
+          {{ document | to_nice_yaml(indent=2) }}
+          {% endfor -%}
+
+      vars:
+        # Parse yaml payload (remove empty documents).
+        _documents: >-
+          {{ slurp_kubeadm_config.content | b64decode
+                                          | from_yaml_all
+                                          | select
+                                          | list }}
+        # Prepare the patch.
+        _update:
+          kubernetesVersion: "v{{ version }}"
+
+        # Process all documents (returns a list of dictionaries).
+        _documents_updated: >-
+          {%- set output = [] -%}
+          {%- for document in _documents -%}
+            {%- if document.kind is defined and document.kind == 'ClusterConfiguration' -%}
+              {{- output.append(document | combine(_update, recursive=true)) -}}
+            {%- else -%}
+              {{- output.append(document) -}}
+            {%- endif -%}
+          {%- endfor -%}
+          {{- output -}}