From 0a5f99cd5e81434fa0e5918627ef2d684e5f1442 Mon Sep 17 00:00:00 2001 From: Norbix Date: Wed, 13 Jul 2022 12:44:57 +0200 Subject: [PATCH 1/7] [WIP][3190] - feat: Expand K8s worker node capabilities with storage from rook --- .../playbooks/roles/kubernetes_master/tasks/master-init.yml | 5 +++++ .../roles/kubernetes_master/templates/kubeadm-config.yml.j2 | 1 + .../roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 | 1 + schema/common/defaults/configuration/kubernetes-node.yml | 1 + schema/common/validation/configuration/kubernetes-node.yml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml b/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml index fa77f08a73..724e915f7f 100644 --- a/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml +++ b/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml @@ -1,4 +1,9 @@ --- +- name: Load default variables for k8s nodes + include_vars: + file: roles/kubernetes_node/vars/main.yml + name: kubernetes_node_vars + - name: Check kubeadm init stat: path: /etc/kubernetes/manifests/kube-apiserver.yaml diff --git a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 index a1f4bd1c1a..98beb40c7d 100644 --- a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 @@ -61,3 +61,4 @@ kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 cgroupDriver: systemd rotateCertificates: true +enableControllerAttachDetach: {{ kubernetes_node_vars.specification.enable_controller_attach_detach }} diff --git a/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 b/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 index 9da0febf17..cf399d62ef 100644 --- a/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 @@ -10,3 +10,4 @@ nodeRegistration: kubeletExtraArgs: enable-controller-attach-detach: "false" node-labels: {{ specification.node_labels }} + enable_controller_attach_detach: "{{ specification.enable_controller_attach_detach }}" diff --git a/schema/common/defaults/configuration/kubernetes-node.yml b/schema/common/defaults/configuration/kubernetes-node.yml index 62cfbdd9dd..f6d6891c6f 100644 --- a/schema/common/defaults/configuration/kubernetes-node.yml +++ b/schema/common/defaults/configuration/kubernetes-node.yml @@ -6,3 +6,4 @@ specification: version: 1.22.4 cni_version: 0.8.7 node_labels: "node-type=epiphany" + enable_controller_attach_detach: false diff --git a/schema/common/validation/configuration/kubernetes-node.yml b/schema/common/validation/configuration/kubernetes-node.yml index 33d71d3ecb..196623f1ce 100644 --- a/schema/common/validation/configuration/kubernetes-node.yml +++ b/schema/common/validation/configuration/kubernetes-node.yml @@ -10,3 +10,5 @@ properties: type: string node_labels: type: string + enable_controller_attach_detach: + type: boolean From f6e24dda03e2a78a724dfb8c53a43f39e6a1fdba Mon Sep 17 00:00:00 2001 From: Norbix Date: Wed, 13 Jul 2022 12:47:06 +0200 Subject: [PATCH 2/7] [3190] - feat: Expand K8s worker node capabilities with storage from rook, again. --- schema/common/validation/configuration/kubernetes-node.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/schema/common/validation/configuration/kubernetes-node.yml b/schema/common/validation/configuration/kubernetes-node.yml index 196623f1ce..385cd6d4a0 100644 --- a/schema/common/validation/configuration/kubernetes-node.yml +++ b/schema/common/validation/configuration/kubernetes-node.yml @@ -12,3 +12,4 @@ properties: type: string enable_controller_attach_detach: type: boolean + From e08bdd7ac04b9d03ea68b6314e1f0af413e57907 Mon Sep 17 00:00:00 2001 From: Norbix Date: Wed, 13 Jul 2022 12:48:45 +0200 Subject: [PATCH 3/7] [WIP][3190] - feat: Expand K8s worker node capabilities with storage from rook, again. --- schema/common/validation/configuration/kubernetes-node.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/schema/common/validation/configuration/kubernetes-node.yml b/schema/common/validation/configuration/kubernetes-node.yml index 385cd6d4a0..196623f1ce 100644 --- a/schema/common/validation/configuration/kubernetes-node.yml +++ b/schema/common/validation/configuration/kubernetes-node.yml @@ -12,4 +12,3 @@ properties: type: string enable_controller_attach_detach: type: boolean - From 802be3eb591e60a984542704ebf275a75903f586 Mon Sep 17 00:00:00 2001 From: Norbix Date: Thu, 14 Jul 2022 16:06:07 +0200 Subject: [PATCH 4/7] [WIP][3190] - feat: Solving issues after code review. --- .../playbooks/roles/kubernetes_master/tasks/master-init.yml | 5 ----- .../roles/kubernetes_master/templates/kubeadm-config.yml.j2 | 2 +- .../roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 | 2 -- schema/common/defaults/configuration/kubernetes-master.yml | 1 + schema/common/defaults/configuration/kubernetes-node.yml | 1 - 5 files changed, 2 insertions(+), 9 deletions(-) diff --git a/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml b/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml index 724e915f7f..fa77f08a73 100644 --- a/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml +++ b/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml @@ -1,9 +1,4 @@ --- -- name: Load default variables for k8s nodes - include_vars: - file: roles/kubernetes_node/vars/main.yml - name: kubernetes_node_vars - - name: Check kubeadm init stat: path: /etc/kubernetes/manifests/kube-apiserver.yaml diff --git a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 index 98beb40c7d..ddb127da55 100644 --- a/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_master/templates/kubeadm-config.yml.j2 @@ -61,4 +61,4 @@ kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 cgroupDriver: systemd rotateCertificates: true -enableControllerAttachDetach: {{ kubernetes_node_vars.specification.enable_controller_attach_detach }} +enableControllerAttachDetach: {{ specification.enable_controller_attach_detach }} diff --git a/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 b/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 index cf399d62ef..d5027362d1 100644 --- a/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 +++ b/ansible/playbooks/roles/kubernetes_node/templates/kubeadm-join-node.yml.j2 @@ -8,6 +8,4 @@ discovery: - sha256:{{ kubernetes_common.kubeadm_cert_hash }} nodeRegistration: kubeletExtraArgs: - enable-controller-attach-detach: "false" node-labels: {{ specification.node_labels }} - enable_controller_attach_detach: "{{ specification.enable_controller_attach_detach }}" diff --git a/schema/common/defaults/configuration/kubernetes-master.yml b/schema/common/defaults/configuration/kubernetes-master.yml index e8d07552ca..fb9da249d9 100644 --- a/schema/common/defaults/configuration/kubernetes-master.yml +++ b/schema/common/defaults/configuration/kubernetes-master.yml @@ -42,6 +42,7 @@ specification: hostname: 127.0.0.1 # change if you want a custom port port: 6443 + enable_controller_attach_detach: false # image_registry_secrets: # - email: emaul@domain.com # name: secretname diff --git a/schema/common/defaults/configuration/kubernetes-node.yml b/schema/common/defaults/configuration/kubernetes-node.yml index f6d6891c6f..62cfbdd9dd 100644 --- a/schema/common/defaults/configuration/kubernetes-node.yml +++ b/schema/common/defaults/configuration/kubernetes-node.yml @@ -6,4 +6,3 @@ specification: version: 1.22.4 cni_version: 0.8.7 node_labels: "node-type=epiphany" - enable_controller_attach_detach: false From f0fe0655009d94e3db0e6d1919403fec191bee6e Mon Sep 17 00:00:00 2001 From: Norbix Date: Thu, 14 Jul 2022 17:03:43 +0200 Subject: [PATCH 5/7] [WIP][3190] - feat: Solving issues after 2nd code review by Arek. --- schema/common/validation/configuration/kubernetes-master.yml | 2 ++ schema/common/validation/configuration/kubernetes-node.yml | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/common/validation/configuration/kubernetes-master.yml b/schema/common/validation/configuration/kubernetes-master.yml index 0f9086bda1..3d4a7cb1b3 100644 --- a/schema/common/validation/configuration/kubernetes-master.yml +++ b/schema/common/validation/configuration/kubernetes-master.yml @@ -144,6 +144,8 @@ properties: - certificates - etcd_args - kubeconfig + enable_controller_attach_detach: + type: boolean required: - version - cni_version diff --git a/schema/common/validation/configuration/kubernetes-node.yml b/schema/common/validation/configuration/kubernetes-node.yml index 196623f1ce..33d71d3ecb 100644 --- a/schema/common/validation/configuration/kubernetes-node.yml +++ b/schema/common/validation/configuration/kubernetes-node.yml @@ -10,5 +10,3 @@ properties: type: string node_labels: type: string - enable_controller_attach_detach: - type: boolean From 7c2da097b0881254e07973ee024fa628080486e4 Mon Sep 17 00:00:00 2001 From: Norbix Date: Sun, 24 Jul 2022 11:58:40 +0200 Subject: [PATCH 6/7] [WIP][3190] - feat: Adding more verbose debugging logic with break point. --- .../tasks/extend-kubeadm-config.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml b/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml index 7a931cc721..e3eea594f7 100644 --- a/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml +++ b/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml @@ -5,6 +5,9 @@ - update is defined fail_msg: Variable 'update' must be defined. +- debug: + var: update + - name: Collect kubeadm-config command: | kubectl get configmap kubeadm-config \ @@ -16,6 +19,11 @@ register: kubeadm_config changed_when: false +- debug: + msg: + - "{{ kubeadm_config }}" + + - name: Extend kubeadm config set_fact: kubeadm_config: >- @@ -24,9 +32,17 @@ original: >- {{ kubeadm_config.stdout | from_yaml }} +- debug: + msg: + - "{{ kubeadm_config }}" + +- name: Pause until you can verify updates to an application were successful + ansible.builtin.pause: + - name: Render /etc/kubeadm/kubeadm-config.yml copy: dest: /etc/kubeadm/kubeadm-config.yml mode: u=rw,go= + backup: yes content: >- {{ kubeadm_config | to_nice_yaml }} From cbbb3bc9b2f7b6c66e70bea2a82c740a0c60a58c Mon Sep 17 00:00:00 2001 From: Norbix Date: Sun, 24 Jul 2022 16:58:14 +0200 Subject: [PATCH 7/7] [WIP][3190] - feat: Adding kubeadm file merge. --- .../kubernetes_common/tasks/extend-kubeadm-config.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml b/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml index e3eea594f7..7f0ad6b608 100644 --- a/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml +++ b/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml @@ -36,8 +36,8 @@ msg: - "{{ kubeadm_config }}" -- name: Pause until you can verify updates to an application were successful - ansible.builtin.pause: +#- name: Pause until you can verify updates to an application were successful +# ansible.builtin.pause: - name: Render /etc/kubeadm/kubeadm-config.yml copy: @@ -46,3 +46,9 @@ backup: yes content: >- {{ kubeadm_config | to_nice_yaml }} + +- name: Merge /etc/kubeadm/kubeadm-config.yml with baseline (2 Kinds) file. + shell: >- + cd /etc/kubeadm && + sed -n -e '/---/,$p' kubeadm-config.yml.* >> kubeadm-config.yml + register: result