diff --git a/docs/ansible/vars.md b/docs/ansible/vars.md index b172f4ada27..f8d040e12ea 100644 --- a/docs/ansible/vars.md +++ b/docs/ansible/vars.md @@ -337,6 +337,13 @@ in the form of dicts of key-value pairs of configuration parameters that will be * *kube_kubeadm_controller_extra_args* * *kube_kubeadm_scheduler_extra_args* +### Kubeadm patches + +When extra flags are not sufficient and there is a need to further customize kubernetes components, +[kubeadm patches](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches) +can be used. +You should use the [`kubeadm_patches` variable](../../roles/kubernetes/kubeadm_common/defaults/main.yml) for that purpose. + ## App variables * *helm_version* - Only supports v3.x. Existing v2 installs (with Tiller) will not be modified and need to be removed manually. diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 522ddc58903..24f896818a7 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -366,11 +366,25 @@ auto_renew_certificates: false # First Monday of each month # auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00" -# kubeadm patches path -kubeadm_patches: - enabled: false - source_dir: "{{ inventory_dir }}/patches" - dest_dir: "{{ kube_config_dir }}/patches" +kubeadm_patches_dir: "{{ kube_config_dir }}/patches" +kubeadm_patches: [] +# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches +# Correspondance with this link +# patchtype = type +# target = target +# suffix -> managed automatically +# extension -> always "yaml" +# kubeadm_patches: +# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration +# type: strategic(default)|json|merge +# patch: +# metadata: +# annotations: +# example.com/test: "true" +# labels: +# example.com/prod_level: "{{ prod_level }}" +# - ... +# Patches are applied in the order they are specified. # Set to true to remove the role binding to anonymous users created by kubeadm remove_anonymous_access: false diff --git a/inventory/sample/patches/kube-controller-manager+merge.yaml b/inventory/sample/patches/kube-controller-manager+merge.yaml deleted file mode 100644 index 3f0fbbcd5e8..00000000000 --- a/inventory/sample/patches/kube-controller-manager+merge.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: kube-controller-manager - annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: '10257' diff --git a/inventory/sample/patches/kube-scheduler+merge.yaml b/inventory/sample/patches/kube-scheduler+merge.yaml deleted file mode 100644 index 00f457237cf..00000000000 --- a/inventory/sample/patches/kube-scheduler+merge.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: kube-scheduler - annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: '10259' diff --git a/roles/kubernetes/control-plane/meta/main.yml b/roles/kubernetes/control-plane/meta/main.yml index 7d793f92f39..7b2cfe3655d 100644 --- a/roles/kubernetes/control-plane/meta/main.yml +++ b/roles/kubernetes/control-plane/meta/main.yml @@ -1,5 +1,6 @@ --- dependencies: + - role: kubernetes/kubeadm_common - role: kubernetes/tokens when: kube_token_auth tags: diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 08450579a7d..62fa2be5b20 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -167,21 +167,6 @@ - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed - not kube_external_ca_mode -- name: Kubeadm | Create directory to store kubeadm patches - file: - path: "{{ kubeadm_patches.dest_dir }}" - state: directory - mode: "0640" - when: kubeadm_patches is defined and kubeadm_patches.enabled - -- name: Kubeadm | Copy kubeadm patches from inventory files - copy: - src: "{{ kubeadm_patches.source_dir }}/" - dest: "{{ kubeadm_patches.dest_dir }}" - owner: "root" - mode: "0644" - when: kubeadm_patches is defined and kubeadm_patches.enabled - - name: Kubeadm | Initialize first control plane node command: >- timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }} diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml index 343724c473c..9609c2f3d85 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -18,7 +18,7 @@ --ignore-preflight-errors=all --allow-experimental-upgrades --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }} - {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %} + {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %} --force register: kubeadm_upgrade # Retry is because upload config sometimes fails @@ -39,7 +39,7 @@ --ignore-preflight-errors=all --allow-experimental-upgrades --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }} - {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %} + {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %} --force register: kubeadm_upgrade # Retry is because upload config sometimes fails diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index ca48a3a9116..9dd5e43764e 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -28,9 +28,9 @@ nodeRegistration: kubeletExtraArgs: cloud-provider: external {% endif %} -{% if kubeadm_patches is defined and kubeadm_patches.enabled %} +{% if kubeadm_patches | length > 0 %} patches: - directory: {{ kubeadm_patches.dest_dir }} + directory: {{ kubeadm_patches_dir }} {% endif %} --- apiVersion: kubeadm.k8s.io/v1beta3 diff --git a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 index cd19b5c2e5f..bc9f3bdf929 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 @@ -31,7 +31,7 @@ nodeRegistration: {% else %} taints: [] {% endif %} -{% if kubeadm_patches is defined and kubeadm_patches.enabled %} +{% if kubeadm_patches | length > 0 %} patches: - directory: {{ kubeadm_patches.dest_dir }} + directory: {{ kubeadm_patches_dir }} {% endif %} diff --git a/roles/kubernetes/kubeadm/meta/main.yml b/roles/kubernetes/kubeadm/meta/main.yml new file mode 100644 index 00000000000..4e727855957 --- /dev/null +++ b/roles/kubernetes/kubeadm/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: kubernetes/kubeadm_common diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 9e01f5fe5fa..cb29e2a6a32 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -83,21 +83,6 @@ mode: "0640" when: ('kube_control_plane' not in group_names) -- name: Kubeadm | Create directory to store kubeadm patches - file: - path: "{{ kubeadm_patches.dest_dir }}" - state: directory - mode: "0640" - when: kubeadm_patches is defined and kubeadm_patches.enabled - -- name: Kubeadm | Copy kubeadm patches from inventory files - copy: - src: "{{ kubeadm_patches.source_dir }}/" - dest: "{{ kubeadm_patches.dest_dir }}" - owner: "root" - mode: "0644" - when: kubeadm_patches is defined and kubeadm_patches.enabled - - name: Join to cluster if needed environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin" diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 index 3b3bc57de46..5016df9c3dc 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 @@ -38,7 +38,7 @@ nodeRegistration: - effect: NoSchedule key: node-role.kubernetes.io/calico-rr {% endif %} -{% if kubeadm_patches is defined and kubeadm_patches.enabled %} +{% if kubeadm_patches | length > 0 %} patches: - directory: {{ kubeadm_patches.dest_dir }} + directory: {{ kubeadm_patches_dir }} {% endif %} diff --git a/roles/kubernetes/kubeadm_common/defaults/main.yml b/roles/kubernetes/kubeadm_common/defaults/main.yml new file mode 100644 index 00000000000..acbcdcf5fae --- /dev/null +++ b/roles/kubernetes/kubeadm_common/defaults/main.yml @@ -0,0 +1,20 @@ +--- +kubeadm_patches_dir: "{{ kube_config_dir }}/patches" +kubeadm_patches: [] +# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches +# Correspondance with this link +# patchtype = type +# target = target +# suffix -> managed automatically +# extension -> always "yaml" +# kubeadm_patches: +# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration +# type: strategic(default)|json|merge +# patch: +# metadata: +# annotations: +# example.com/test: "true" +# labels: +# example.com/prod_level: "{{ prod_level }}" +# - ... +# Patches are applied in the order they are specified. diff --git a/roles/kubernetes/kubeadm_common/tasks/main.yml b/roles/kubernetes/kubeadm_common/tasks/main.yml new file mode 100644 index 00000000000..0f8d3b0a0af --- /dev/null +++ b/roles/kubernetes/kubeadm_common/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Kubeadm | Create directory to store kubeadm patches + file: + path: "{{ kubeadm_patches_dir }}" + state: directory + mode: "0640" + when: kubeadm_patches | length > 0 + +- name: Kubeadm | Copy kubeadm patches from inventory files + copy: + content: "{{ item.patch | to_yaml }}" + dest: "{{ kubeadm_patches_dir }}/{{ item.target }}{{ suffix }}+{{ item.type | d('strategic') }}.yaml" + owner: "root" + mode: "0644" + loop: "{{ kubeadm_patches }}" + loop_control: + index_var: suffix diff --git a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml index 4f35d2f8763..7a27ce56610 100644 --- a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml +++ b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml @@ -27,3 +27,20 @@ containerd_registries_mirrors: skip_verify: true calico_datastore: "etcd" + +# Test kubeadm patches +kubeadm_patches: + - target: kube-apiserver + patch: + metadata: + annotations: + example.com/test: "true" + labels: + example.com/prod_level: "prep" + - target: kube-controller-manager + patch: + metadata: + annotations: + example.com/test: "false" + labels: + example.com/prod_level: "prep"