From 243ca5d08fa73a009760c6ce486e7d86c9481398 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 20 Dec 2023 09:59:02 +0100 Subject: [PATCH 001/103] Add test case for calico using etcd datastore (#10722) * Add multinode ci layout * Add test case for calico using etcd datastore --- .gitlab-ci/packet.yml | 5 ++++ docs/test_cases.md | 4 ++- .../roles/packet-ci/templates/inventory.j2 | 10 +++++++ .../roles/packet-ci/vars/main.yml | 1 + .../packet_ubuntu22-calico-etcd-datastore.yml | 29 +++++++++++++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/files/packet_ubuntu22-calico-etcd-datastore.yml diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index d1c5ba0af6e..de61a1f60dd 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -61,6 +61,11 @@ packet_ubuntu22-calico-all-in-one: extends: .packet_pr when: on_success +packet_ubuntu22-calico-etcd-datastore: + stage: deploy-part2 + extends: .packet_pr + when: on_success + packet_centos7-flannel-addons-ha: extends: .packet_pr stage: deploy-part2 diff --git a/docs/test_cases.md b/docs/test_cases.md index 8bfc7bd58cb..d5aef62f2a7 100644 --- a/docs/test_cases.md +++ b/docs/test_cases.md @@ -1,6 +1,6 @@ # Node Layouts -There are five node layout types: `default`, `separate`, `ha`, `scale`, and `all-in-one`. +There are six node layout types: `default`, `separate`, `ha`, `scale`, `all-in-one`, and `multinode`. `default` is a non-HA two nodes setup with one separate `kube_node` and the `etcd` group merged with the `kube_control_plane`. @@ -18,6 +18,8 @@ never actually deployed, but certificates are generated for them. `all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged. +`multinode` layout consists of two separate `kube_node` and a merged single `etcd+kube_control_plane` node. + Note, the canal network plugin deploys flannel as well plus calico policy controller. ## Test cases diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 index e2b1c9b8c2d..da7e749691c 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 @@ -81,6 +81,16 @@ instance-2 [broken_etcd] instance-1 etcd_member_name=etcd2 instance-2 etcd_member_name=etcd3 +{% elif mode == "multinode" %} +[kube_control_plane] +instance-1 + +[etcd] +instance-1 + +[kube_node] +instance-2 +instance-3 {% endif %} [k8s_cluster:children] diff --git a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml index 14cf5917c87..5cfe561b707 100644 --- a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml @@ -7,6 +7,7 @@ _vm_count_dict: ha-recover: 3 ha-recover-noquorum: 3 all-in-one: 1 + multinode: 3 default: 2 vm_count: "{{ _vm_count_dict[mode | d('default')] }}" diff --git a/tests/files/packet_ubuntu22-calico-etcd-datastore.yml b/tests/files/packet_ubuntu22-calico-etcd-datastore.yml new file mode 100644 index 00000000000..c95a7602fd9 --- /dev/null +++ b/tests/files/packet_ubuntu22-calico-etcd-datastore.yml @@ -0,0 +1,29 @@ +--- +# Instance settings +cloud_image: ubuntu-2204 +mode: multinode +vm_memory: 1600Mi + +# Kubespray settings +auto_renew_certificates: true + +# Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko +kube_proxy_mode: iptables +enable_nodelocaldns: False + +containerd_registries: + "docker.io": "https://mirror.gcr.io" + +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://mirror.gcr.io + capabilities: ["pull", "resolve"] + skip_verify: false + - prefix: 172.19.16.11:5000 + mirrors: + - host: http://172.19.16.11:5000 + capabilities: ["pull", "resolve", "push"] + skip_verify: true + +calico_datastore: "etcd" From da9b34d1b0167f981307b2b0ba298941b317662c Mon Sep 17 00:00:00 2001 From: my-git9 Date: Thu, 21 Dec 2023 16:52:07 +0800 Subject: [PATCH 002/103] docs: improve format for offline/README.md (#10731) Signed-off-by: xin.li --- contrib/offline/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/offline/README.md b/contrib/offline/README.md index a7a87d8f021..a0e560ec8cc 100644 --- a/contrib/offline/README.md +++ b/contrib/offline/README.md @@ -5,7 +5,9 @@ Container image collecting script for offline deployment This script has two features: + (1) Get container images from an environment which is deployed online. + (2) Deploy local container registry and register the container images to the registry. Step(1) should be done online site as a preparation, then we bring the gotten images From c3315ac742d572e89e702c58859d7614cee7382c Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 21 Dec 2023 09:52:14 +0100 Subject: [PATCH 003/103] systemd-resolved: use a drop-in for kubespray dns (#10732) This avoid needlessly overriding things and make cleanup easier. Also simplifies the template a bit. --- .../preinstall/tasks/0061-systemd-resolved.yml | 10 ++++++++-- roles/kubernetes/preinstall/templates/resolved.conf.j2 | 10 ++-------- roles/reset/tasks/main.yml | 1 + 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml b/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml index 3811358629c..9edec2e6488 100644 --- a/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml +++ b/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml @@ -1,8 +1,14 @@ --- -- name: Write resolved.conf +- name: Create systemd-resolved drop-in directory + file: + state: directory + name: /etc/systemd/resolved.conf.d/ + mode: 0755 + +- name: Write Kubespray DNS settings to systemd-resolved template: src: resolved.conf.j2 - dest: /etc/systemd/resolved.conf + dest: /etc/systemd/resolved.conf.d/kubespray.conf owner: root group: root mode: 0644 diff --git a/roles/kubernetes/preinstall/templates/resolved.conf.j2 b/roles/kubernetes/preinstall/templates/resolved.conf.j2 index 7c47b0b288f..edafbf9b9d3 100644 --- a/roles/kubernetes/preinstall/templates/resolved.conf.j2 +++ b/roles/kubernetes/preinstall/templates/resolved.conf.j2 @@ -1,21 +1,15 @@ [Resolve] -{% if dns_early is sameas true and dns_late is sameas false %} -#DNS= -{% else %} +{% if not dns_early and dns_late %} DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }} {% endif %} FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }} -{% if remove_default_searchdomains is sameas true and searchdomains|default([])|length != 0 %} +{% if remove_default_searchdomains and searchdomains|default([])|length != 0 %} Domains={{ searchdomains|default([]) | join(' ') }} {% else %} Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} {% endif %} -#LLMNR=no -#MulticastDNS=no DNSSEC=no Cache=no-negative {% if systemd_resolved_disable_stub_listener | bool %} DNSStubListener=no -{% else %} -#DNSStubListener=yes {% endif %} diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 53fb9f44bb6..84be011a516 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -314,6 +314,7 @@ - /etc/dnsmasq.d - /etc/dnsmasq.conf - /etc/dnsmasq.d-available + - /etc/systemd/resolved.conf.d/kubespray.conf - /etc/etcd.env - /etc/calico - /etc/NetworkManager/conf.d/calico.conf From 565eab901bfd11c033296b60072536b7199d10f3 Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Thu, 21 Dec 2023 17:01:12 +0800 Subject: [PATCH 004/103] remove containerd registries (#10738) --- docs/containerd.md | 2 ++ roles/container-engine/containerd/defaults/main.yml | 3 --- tests/common/_docker_hub_registry_mirror.yml | 5 ----- tests/files/packet_ubuntu22-calico-all-in-one.yml | 3 --- 4 files changed, 2 insertions(+), 11 deletions(-) diff --git a/docs/containerd.md b/docs/containerd.md index 112c0fc1efb..c9d18051a6e 100644 --- a/docs/containerd.md +++ b/docs/containerd.md @@ -44,6 +44,8 @@ containerd_registries_mirrors: image_command_tool: crictl ``` +The `containerd_registries` and `containerd_insecure_registries` configs are deprecated. + ### Containerd Runtimes Containerd supports multiple runtime configurations that can be used with diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml index 2fb4afe900b..fd36c50de06 100644 --- a/roles/container-engine/containerd/defaults/main.yml +++ b/roles/container-engine/containerd/defaults/main.yml @@ -48,9 +48,6 @@ containerd_metrics_address: "" containerd_metrics_grpc_histogram: false -containerd_registries: - "docker.io": "https://registry-1.docker.io" - containerd_registries_mirrors: - prefix: docker.io mirrors: diff --git a/tests/common/_docker_hub_registry_mirror.yml b/tests/common/_docker_hub_registry_mirror.yml index db521d67952..4280f9d8e58 100644 --- a/tests/common/_docker_hub_registry_mirror.yml +++ b/tests/common/_docker_hub_registry_mirror.yml @@ -5,11 +5,6 @@ docker_registry_mirrors: containerd_grpc_max_recv_message_size: 16777216 containerd_grpc_max_send_message_size: 16777216 -containerd_registries: - "docker.io": - - "https://mirror.gcr.io" - - "https://registry-1.docker.io" - containerd_registries_mirrors: - prefix: docker.io mirrors: diff --git a/tests/files/packet_ubuntu22-calico-all-in-one.yml b/tests/files/packet_ubuntu22-calico-all-in-one.yml index 4c5b5d711d8..614cbeabccb 100644 --- a/tests/files/packet_ubuntu22-calico-all-in-one.yml +++ b/tests/files/packet_ubuntu22-calico-all-in-one.yml @@ -11,9 +11,6 @@ auto_renew_certificates: true kube_proxy_mode: iptables enable_nodelocaldns: False -containerd_registries: - "docker.io": "https://mirror.gcr.io" - containerd_registries_mirrors: - prefix: docker.io mirrors: From c3b674526da9d77a8a459c25cd6ca51936e78ba2 Mon Sep 17 00:00:00 2001 From: Andrei Costescu Date: Thu, 21 Dec 2023 16:16:34 +0100 Subject: [PATCH 005/103] Fix modprobe module on Flatcar (#10678) * Fix modprobe module on Flatcar * Add todo about upstream issue report --- roles/kubernetes/node/tasks/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index bb3d4238c72..9e15b16d9ba 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -68,11 +68,16 @@ changed_when: false check_mode: no +# TODO: Remove once upstream issue is fixed +# https://github.com/ansible-collections/community.general/issues/7717 - name: Verify br_netfilter module path exists file: - path: /etc/modules-load.d + path: "{{ item }}" state: directory mode: 0755 + loop: + - /etc/modules-load.d + - /etc/modprobe.d - name: Enable br_netfilter module community.general.modprobe: From ee2dd4fd284d04334e3b82e75016419bb92ecd5f Mon Sep 17 00:00:00 2001 From: my-git9 Date: Thu, 21 Dec 2023 23:45:09 +0800 Subject: [PATCH 006/103] support other other runtime tool for manage-offline-container-images (#10740) Signed-off-by: xin.li --- .../manage-offline-container-images.sh | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/contrib/offline/manage-offline-container-images.sh b/contrib/offline/manage-offline-container-images.sh index 40ff2c2bd9c..2098eea639c 100755 --- a/contrib/offline/manage-offline-container-images.sh +++ b/contrib/offline/manage-offline-container-images.sh @@ -23,8 +23,8 @@ function create_container_image_tar() { mkdir ${IMAGE_DIR} cd ${IMAGE_DIR} - sudo docker pull registry:latest - sudo docker save -o registry-latest.tar registry:latest + sudo ${runtime} pull registry:latest + sudo ${runtime} save -o registry-latest.tar registry:latest for image in ${IMAGES} do @@ -32,7 +32,7 @@ function create_container_image_tar() { set +e for step in $(seq 1 ${RETRY_COUNT}) do - sudo docker pull ${image} + sudo ${runtime} pull ${image} if [ $? -eq 0 ]; then break fi @@ -42,7 +42,7 @@ function create_container_image_tar() { fi done set -e - sudo docker save -o ${FILE_NAME} ${image} + sudo ${runtime} save -o ${FILE_NAME} ${image} # NOTE: Here removes the following repo parts from each image # so that these parts will be replaced with Kubespray. @@ -95,16 +95,16 @@ function register_container_images() { sed -i s@"HOSTNAME"@"${LOCALHOST_NAME}"@ ${TEMP_DIR}/registries.conf sudo cp ${TEMP_DIR}/registries.conf /etc/containers/registries.conf else - echo "docker package(docker-ce, etc.) should be installed" + echo "runtime package(docker-ce, podman, nerctl, etc.) should be installed" exit 1 fi tar -zxvf ${IMAGE_TAR_FILE} - sudo docker load -i ${IMAGE_DIR}/registry-latest.tar + sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar set +e - sudo docker container inspect registry >/dev/null 2>&1 + sudo ${runtime} container inspect registry >/dev/null 2>&1 if [ $? -ne 0 ]; then - sudo docker run --restart=always -d -p 5000:5000 --name registry registry:latest + sudo ${runtime} run --restart=always -d -p 5000:5000 --name registry registry:latest fi set -e @@ -112,8 +112,8 @@ function register_container_images() { file_name=$(echo ${line} | awk '{print $1}') raw_image=$(echo ${line} | awk '{print $2}') new_image="${LOCALHOST_NAME}:5000/${raw_image}" - org_image=$(sudo docker load -i ${IMAGE_DIR}/${file_name} | head -n1 | awk '{print $3}') - image_id=$(sudo docker image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//) + org_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1 | awk '{print $3}') + image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//) if [ -z "${file_name}" ]; then echo "Failed to get file_name for line ${line}" exit 1 @@ -130,9 +130,9 @@ function register_container_images() { echo "Failed to get image_id for file ${file_name}" exit 1 fi - sudo docker load -i ${IMAGE_DIR}/${file_name} - sudo docker tag ${image_id} ${new_image} - sudo docker push ${new_image} + sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} + sudo ${runtime} tag ${image_id} ${new_image} + sudo ${runtime} push ${new_image} done <<< "$(cat ${IMAGE_LIST})" echo "Succeeded to register container images to local registry." @@ -143,6 +143,18 @@ function register_container_images() { echo "- quay_image_repo" } +# get runtime command +if command -v nerdctl 1>/dev/null 2>&1; then + runtime="nerdctl" +elif command -v podman 1>/dev/null 2>&1; then + runtime="podman" +elif command -v docker 1>/dev/null 2>&1; then + runtime="docker" +else + echo "No supported container runtime found" + exit 1 +fi + if [ "${OPTION}" == "create" ]; then create_container_image_tar elif [ "${OPTION}" == "register" ]; then From aea150e5dc244e933c6d5e2aee35ffb7ffe614a9 Mon Sep 17 00:00:00 2001 From: Mohamed Omar Zaian Date: Thu, 21 Dec 2023 17:30:45 +0100 Subject: [PATCH 007/103] [kubernetes] Make kubernetes v1.28.5 default (#10739) * Add hashes for kubernetes 1.29.0, 1.28.5, 1.27.9, 1.26.12 --- README.md | 2 +- .../group_vars/k8s_cluster/k8s-cluster.yml | 2 +- .../defaults/main/checksums.yml | 48 +++++++++++++++++++ .../kubespray-defaults/defaults/main/main.yml | 2 +- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d2cf298f027..b4a721da139 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Note: Upstart/SysV init based OS types are not supported. ## Supported Components - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.4 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.5 - [etcd](https://github.com/etcd-io/etcd) v3.5.9 - [docker](https://www.docker.com/) v20.10 (see note) - [containerd](https://containerd.io/) v1.7.11 diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index bb2250a345f..8153a04cde7 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.28.4 +kube_version: v1.28.5 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index e42029d8bc7..7f046f475a4 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -73,11 +73,14 @@ crio_archive_checksums: # Kubernetes versions above Kubespray's current target version are untested and should be used with caution. kubelet_checksums: arm: + v1.29.0: 0 + v1.28.5: 0 v1.28.4: 0 v1.28.3: 0 v1.28.2: 0 v1.28.1: 0 v1.28.0: 0 + v1.27.9: 0 v1.27.8: 0 v1.27.7: 0 v1.27.6: 0 @@ -87,6 +90,7 @@ kubelet_checksums: v1.27.2: 0 v1.27.1: 0 v1.27.0: 0 + v1.26.12: 9bfa6d78d4f6a865d7a1731624d5aafdc69dff54139eed6c2f076158a67c904a v1.26.11: a7bbe545ab23b068d5e25262bf17a10039edb5c073ba896d036af12afb5fbf9c v1.26.10: 3130ae14dda009f92296045d9cc2a1f566fa7d85905d0f6a8db47b05496c5d53 v1.26.9: 739c62a6801d935477121614ee3a2ef6deba78ecd088ae5477c3f18bf19d68c8 @@ -100,11 +104,14 @@ kubelet_checksums: v1.26.1: fe940be695f73c03275f049cb17f2bf2eb137014334930ce5c6de12573c1f21f v1.26.0: cabf702fc542fcbb1173c713f1cbec72fd1d9ded36cdcdbbd05d9c308d8360d1 arm64: + v1.29.0: 0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16 + v1.28.5: 28ddb696eb6e076f2a2f59ccaa2e409785a63346e5bda819717c6e0f58297702 v1.28.4: bf203989dd9b3987b8a0d2331dcce6319f834b57df810fafba5a4805d54823ac v1.28.3: 64f56e9c55183919153fe59df2c9015dff09c56de13a3cbccc0f04a95b76dab9 v1.28.2: 32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577 v1.28.1: 9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c v1.28.0: 05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9 + v1.27.9: 8a14bc3739f5ca3b23d08301c2e769ee58c8d1cecb7243b46b1c098ae77effd7 v1.27.8: 71849182ceb018dc084f499ad28b7b1afb7f23e35ccaf8421941dd5dafef0d4c v1.27.7: ed5bfa48ee64d5e6cf23ed9fc03ea0593021839429fdc1ea7cc2ebf3f11b6491 v1.27.6: be579ef4e8fa3e1de9d40a77e4d35d99e535a293f66bf3038cbea9cf803d11e5 @@ -114,6 +121,7 @@ kubelet_checksums: v1.27.2: 810cd9a611e9f084e57c9ee466e33c324b2228d4249ff38c2588a0cc3224f10d v1.27.1: dbb09d297d924575654db38ed2fc627e35913c2d4000c34613ac6de4995457d0 v1.27.0: 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157 + v1.26.12: d4406ed5bfd12768c03fc4fbe011a01e5c91b74d1d4b526fe3ac320d13295ffb v1.26.11: ff8940394446028e75a2b8155e22eccf635f6a128f45dee41e293493d2743d17 v1.26.10: ddebcc1af7f203a2ee3d80dad0baaf84a4680748839f5583b39cbce4b8afa7f2 v1.26.9: f6b1dcee9960ffe6b778dc91cabef8ce4a7bd06c76378ef2784232709eace6a5 @@ -127,11 +135,14 @@ kubelet_checksums: v1.26.1: f4b514162b52d19909cf0ddf0b816d8d7751c5f1de60eda90cd84dcccc56c399 v1.26.0: fb033c1d079cac8babb04a25abecbc6cc1a2afb53f56ef1d73f8dc3b15b3c09e amd64: + v1.29.0: e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05 + v1.28.5: bf37335da58182783a8c63866ec1f895b4c436e3ed96bdd87fe3f8ae8004ba1d v1.28.4: db2a473b73c3754d4011590f2f0aa877657608499590c6b0f8b40bec96a3e9ba v1.28.3: a3a058b4ba30da01ffe1801cd38fcad58a9022a2d39e080b4b2e0e9749a75ad5 v1.28.2: 17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185 v1.28.1: 2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3 v1.28.0: bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981 + v1.27.9: ede60eea3acbac3f35dbb23d7b148f45cf169ebbb20af102d3ce141fc0bac60c v1.27.8: 2e0557b38c5b9a1263eed25a0b84d741453ed9c0c7bd916f80eadaf7edfb7784 v1.27.7: 236bc8bc22c52e914d3364c23e273628c63e193365b6a43b8cb013716c1cd2f5 v1.27.6: daa42f9b6f5e2176bbce0d24d89a05613000630bcddec1fafd2a8d42a523ce9d @@ -141,6 +152,7 @@ kubelet_checksums: v1.27.2: a0d12afcab3b2836de4a427558d067bebdff040e9b306b0512c93d9d2a066579 v1.27.1: cb2845fff0ce41c400489393da73925d28fbee54cfeb7834cd4d11e622cbd3a7 v1.27.0: 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29 + v1.26.12: aed0a351b01f1e6a84a0992ef1265bb0c9994b900162c075df58d0d02517d3df v1.26.11: a62953f20fa9fedff50c6c5423e68981e3382d92cf04174d5bca5f4d084de0c5 v1.26.10: 4c27b3a9f332a6762f7240d0784c64775d4db5a1b881eeae05c4561d06c267ec v1.26.9: baa2b021ab2f90c342518e2b8981a18de7e1e6b33f11c57e3ff23d40364877a8 @@ -154,11 +166,14 @@ kubelet_checksums: v1.26.1: 8b99dd73f309ca1ac4005db638e82f949ffcfb877a060089ec0e729503db8198 v1.26.0: b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5 ppc64le: + v1.29.0: 67f09f866d3e4aee8211ce9887ec8bc427b188474a882a7af999fc0fee939028 + v1.28.5: ae9fe81804ba67ee81e8a5fe1dc18fe285267764c61f831886a25245a11d8528 v1.28.4: d79c97811fb10c1b1f48b69573f1164f108630631d9dba0d991fe924bd305f20 v1.28.3: f20cfb8c9de73cdc66fbbecd03bb936ce57fe86ebced8ea93aa64ebda0235c21 v1.28.2: 79f568ac700d29f88d669c6b6a09adb3b726bdd13c10aa0839cbc70b414372e5 v1.28.1: 547fc76f0c1d78352fad841ebeacd387fe48750b2648565dfd49197621622fbb v1.28.0: 22de59965f2d220afa24bf04f4c6d6b65a4bb1cd80756c13381973b1ac3b4578 + v1.27.9: f270051c9b0f36da10a5d27011783be042edd396e8c729709c2396f29b72b6d2 v1.27.8: 2354fdb19b5018cabe43fde1979965686afd3c95b75531e678a0064c4a30b4e9 v1.27.7: 9154a7b5d5793fb8a05cba0d309ddfd975409d0e0686be19cf2feca172f06162 v1.27.6: 1001da3586a3f868c371aefde991af94ca780ec1599c8a969390ba105aaf9dcb @@ -168,6 +183,7 @@ kubelet_checksums: v1.27.2: 3af92edd687f7932e7fce877944dfe5efa437bf5f171fc8331725c631a1a86ef v1.27.1: 7a800b9539beaba0b5d6357070a40fb3c4d216c2ad6693b15f9b1307b1c99e1f v1.27.0: 17c061a9f7919697ac71c151c19337f65b86f59f59441687ac92e977d851c75b + v1.26.12: c740ea116206a3e9420b1ab810991c21910abeea947da1b295c2454700cdbe44 v1.26.11: 408e47e33159551ebeeb6f4ef836bd7d07fc92d16ca6d662c85ea8fa0fe34a8e v1.26.10: 5cc1046c8f9188dc6d5ec22200eeed7c8c1ae88a23ce5d79e8dfaf1ff61f2bf8 v1.26.9: e87a83c1ca74e55cea51eda53d29324de7fb7f9330c266ea1f2e270fe0f9b677 @@ -182,11 +198,14 @@ kubelet_checksums: v1.26.0: df13099611f4eada791e5f41948ef94f7b52c468dff1a6fcad54f4c56b467a7f kubectl_checksums: arm: + v1.29.0: a2388eb458d07ec734e4fa02fd0147456a1922a7d6b8e67a32db9d64a4d7621c + v1.28.5: 0819c9d0ea66a1e20d74d9a455090e1f67fe07d671866be342ab55532203f4bc v1.28.4: 835ef8d72f8dec4493b855ddd8e4163f107053496d923c89c216489a45757df6 v1.28.3: b252ec9e97abde80fe067eb215a1acb69a8c83022cba897fd2c4d387bd45f5ca v1.28.2: 6576aa70413ff00c593a07b549b8b9d9e5ef73c42bb39ab4af475e0fdb540613 v1.28.1: eaa05dab1bffb8593d8e5caa612530ee5c914ee2be73429b7ce36c3becad893f v1.28.0: 372c4e7bbe98c7067c4b7820c4a440c931ad77f7cb83d3237b439ca3c14d3d37 + v1.27.9: 89b76aa415018377f2c5fc33fc4d45f4997cc63677336f1768ee8a11593515ce v1.27.8: 2f2936f950beb3f08ee0e45fbf80d020163829b95aa11c99ec726ee1a922329c v1.27.7: 2a387ade64cd4cf90f002ae8c0e6f60250d26d1ee2cce6051f559430a44352ea v1.27.6: 3a34a38908a9d0f85dc531cc1c49061ceeaa2ab742382d891d9fc7bf8dc53b8c @@ -196,6 +215,7 @@ kubectl_checksums: v1.27.2: 7792f5630543c0af84f444521ee6113da5ae00f2b50872d57324aa725a5341c5 v1.27.1: fe704e355bf2c5f69964cd12772687535a11a5e9ec0baf4f27e0a8fb156bc615 v1.27.0: 288470e3eb89a2f55273d753ce6674dfb00e732f2971428acb964810aa726188 + v1.26.12: cf788244170a71dc6540abccdff76327a48d8c5fbc88ffde9e1ebaa4616f8d2e v1.26.11: 34cc371daef6e42aea27117aa643619ab870d1cb61995530fc9bcb992e3caf10 v1.26.10: d26ab68b4884eec4dcadd0f564ba197d48f21aebed509b9b1541d31f2aaf5890 v1.26.9: 8e020ffe72dd4c8694ee5e9f124833ca302a2341fa046650482b38ddb189d1fd @@ -209,11 +229,14 @@ kubectl_checksums: v1.26.1: e067d59ac19e287026b5c2b75a1077b1312ba82ad64ee01dff2cdafd57720f39 v1.26.0: 8eef310d0de238c582556d81ab8cbe8d6fca3c0e43ee337a905dcdd3578f9dda arm64: + v1.29.0: 8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7 + v1.28.5: f87fe017ae3ccfd93df03bf17edd4089672528107f230563b8c9966909661ef2 v1.28.4: edf1e17b41891ec15d59dd3cc62bcd2cdce4b0fd9c2ee058b0967b17534457d7 v1.28.3: 06511f03e34d8ee350bd55717845e27ebec3116526db7c60092eeb33a475a337 v1.28.2: ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53 v1.28.1: 46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6 v1.28.0: f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c + v1.27.9: bda475539fdeda9d8a85a84b967af361af264d0826c121b23b0b62ee9b00cd2d v1.27.8: 97ed6739e2803e63fd2d9de78be22d5ba6205bb63179a16ec773063526525a8e v1.27.7: 61fc334f2c0290270e43fb8a1d4ff07e8cec5642d5a123eb7ab66a134b04ae83 v1.27.6: 7322a6f600de6d0d06cf333bdc24cd2a340bba12920b0c2385c97884c808c810 @@ -223,6 +246,7 @@ kubectl_checksums: v1.27.2: 1b0966692e398efe71fe59f913eaec44ffd4468cc1acd00bf91c29fa8ff8f578 v1.27.1: fd3cb8f16e6ed8aee9955b76e3027ac423b6d1cc7356867310d128082e2db916 v1.27.0: f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc + v1.26.12: b9cfdb6c9a53146330d5694e711c48febb7ec022aea8d16a51b5e12d761580cc v1.26.11: 4a6d2b7204af3cf84cd0e2c670fbb211501050c9a288de49de3c6363d4e0a63e v1.26.10: 5752e3908fa1d338eb1fa99a6f39c6a4c27b065cb459da84e35c4ec718879f14 v1.26.9: f945c63220b393ddf8df67d87e67ff74b7f56219a670dee38bc597a078588e90 @@ -236,11 +260,14 @@ kubectl_checksums: v1.26.1: 4027cb0a2840bc14ec3f18151b3360dd2d1f6ce730ed5ac28bd846c17e7d73f5 v1.26.0: 79b14e4ddada9e81d2989f36a89faa9e56f8abe6e0246e7bdc305c93c3731ea4 amd64: + v1.29.0: 0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5 + v1.28.5: 2a44c0841b794d85b7819b505da2ff3acd5950bd1bcd956863714acc80653574 v1.28.4: 893c92053adea6edbbd4e959c871f5c21edce416988f968bec565d115383f7b8 v1.28.3: 0c680c90892c43e5ce708e918821f92445d1d244f9b3d7513023bcae9a6246d1 v1.28.2: c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 v1.28.1: e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1 v1.28.0: 4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce + v1.27.9: d0caae91072297b2915dd65f6ef3055d27646dce821ec67d18da35ba9a8dc85b v1.27.8: 027b3161e99fa0a7fa529e8f17f73ee2c0807c81c721ca7cf307f6b41c17bc57 v1.27.7: e5fe510ba6f421958358d3d43b3f0b04c2957d4bc3bb24cf541719af61a06d79 v1.27.6: 2b7adb71c8630904da1b94e262c8c3c477e9609b3c0ed8ae1213a1e156ae38dd @@ -250,6 +277,7 @@ kubectl_checksums: v1.27.2: 4f38ee903f35b300d3b005a9c6bfb9a46a57f92e89ae602ef9c129b91dc6c5a5 v1.27.1: 7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d v1.27.0: 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87 + v1.26.12: 8e6af8d68e7b9d2a1eb43255c0da793276e549a34a2b9c3c87a9c26438e7fd71 v1.26.11: 27c34a0870230d9dd723e1e01114634e396cd2a3d25ced263b769a4bd53e4edd v1.26.10: 93ad44b4072669237247bfbc171be816f08e7e9e4260418d2cfdd0da1704ae86 v1.26.9: 98ea4a13895e54ba24f57e0d369ff6be0d3906895305d5390197069b1da12ae2 @@ -263,11 +291,14 @@ kubectl_checksums: v1.26.1: d57be22cfa25f7427cfb538cfc8853d763878f8b36c76ce93830f6f2d67c6e5d v1.26.0: b6769d8ac6a0ed0f13b307d289dc092ad86180b08f5b5044af152808c04950ae ppc64le: + v1.29.0: ea926d8cf25e2ce982ff5c375da32b51ccbd122b721b1bc4a32f52a9a0d073ab + v1.28.5: 4448a9f95421cbe69726aa4d2967d706bc43466b9c656c7425b55431b1c20dd4 v1.28.4: 816ca2cef39c0d1ac8ad60c05ae6f6ea5c4a0ca33748240bd1f019381244ca23 v1.28.3: 2b7331a91f558a748167672c18458aa205d4d6d2794654dfd308942e9a376ca4 v1.28.2: 87cca30846fec99a4fbea122b21e938717b309631bd2220de52049fce30d2e81 v1.28.1: 81b45c27abbdf2be6c5203dfccfd76ded1ac273f9f7672e6dcdf3440aa191324 v1.28.0: 7a9dcb4c75b33b9dac497c1a756b1f12c7c63f86fc0f321452360fbe1a79ce0f + v1.27.9: 2464d947370b8902e1245b0a75a4ecf55fe2aeee5bc87f2add7da00b73535a59 v1.27.8: e25a09dea99192ff43ee13af61bfadd7c79eb538dc8e85376b6c590b4d471204 v1.27.7: a60da56f856bc5b224f0be71b03443d5a0fb69424f31cd86f069cc79c13b2870 v1.27.6: f3ed7752a20dbae271eeff9e9d109381e3ed6772853b5c84dc8a7476bbad847c @@ -277,6 +308,7 @@ kubectl_checksums: v1.27.2: efee037a276f72c77cc230194d7dadf943a5778be46b7985edeb414d27894266 v1.27.1: 440bcfd9611319f3d9e5d4fa4cdee2421cdf80c01fad223934d9a9b640673d75 v1.27.0: daa9f1d4fe3f217de2546bca4ac14601f34b34a25c1f571f1e44eb313aee1385 + v1.26.12: f977850aff89f7e148fbe940aa84faf7e1abbf1936527c79fc111a6f759b73b0 v1.26.11: f2c577e75277415203e699ca7bc08eb640a0b8211a0366926122e3ac4f98997c v1.26.10: 24674d7ce1ccef207d127bd952f6f40f951d7682cfa603f6989ed80e91dcaafc v1.26.9: bcb287f24a30bd7ef27bc36dc4f896aba3f1091f947afde73576fbd81af65cc5 @@ -291,11 +323,14 @@ kubectl_checksums: v1.26.0: 9e2b2a03ee5fc726ebd39b6a09498b6508ade8831262859c280d39df33f8830d kubeadm_checksums: arm: + v1.29.0: 0 + v1.28.5: 0 v1.28.4: 0 v1.28.3: 0 v1.28.2: 0 v1.28.1: 0 v1.28.0: 0 + v1.27.9: 0 v1.27.8: 0 v1.27.7: 0 v1.27.6: 0 @@ -305,6 +340,7 @@ kubeadm_checksums: v1.27.2: 0 v1.27.1: 0 v1.27.0: 0 + v1.26.12: a15b212a2ec393b229d35eb0900b287c4a4c52e1993ef69eadcce1105d5c2bc9 v1.26.11: 3683f52861f89c3cf48ea7754fd2e81ad30a51bef57d056b2663e5e9139870b6 v1.26.10: b59433ebaddf5c5d816033304989f9dd2e0b863650532cd195c1d41c0f973f14 v1.26.9: a6841e7e554407776e4d0fc83306756ad1836d1f92d6d5cce1055eee1999732a @@ -318,11 +354,14 @@ kubeadm_checksums: v1.26.1: 0dbd0a197013a3fdc5cb3e012fa8b0d50f38fd3dda56254f4648e08ac867fb60 v1.26.0: 3368537a5e78fdbfa3cbcae0d19102372a0f4eb6b6a78e7b6b187d5db86d6c9e arm64: + v1.29.0: bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980 + v1.28.5: 22bb6b3377204e93d008f33ac4924d77adca1478f1ae3b515c03476ba54f1adc v1.28.4: a4422780020954436b8e76ab1c59b68c5581a54432dd3e566c4709bb40c8d4f9 v1.28.3: dcb37d78ccdfe9d8dd6f100e188ddc6e3f5570d0c49db68470073683b453a1e7 v1.28.2: 010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff v1.28.1: 7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8 v1.28.0: b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab + v1.27.9: d3d022842b0b8e4661222e8873249f5acafdbef52fd1bfb98152a582352b3c40 v1.27.8: 0d0f5b2781d663d314e785d14361aa5a09cfaf6e1694aa3cc731f4f06342ec13 v1.27.7: 46d7c43532233906919a53ee0e03ab04ab9e08514392d17a86f058e0364cda4b v1.27.6: faec35315203913b835e9b789d89001a05e072943c960bcf4de1e331d08e10c8 @@ -332,6 +371,7 @@ kubeadm_checksums: v1.27.2: 8f01f363f7c7f92de2f2276124a895503cdc5a60ff549440170880f296b087eb v1.27.1: 024a59cd6fc76784b597c0c1cf300526e856e8c9fefa5fa7948158929b739551 v1.27.0: acd805c6783b678ee0068b9dd8165bbfd879c345fd9c25d6a978dbc965f48544 + v1.26.12: 2dee03d460e8a1b3f30e5ef48ce94f5eeb4ff4550e65860e6e2d94b368c2cde6 v1.26.11: a13318c1493e58a9f7c4359c79443f3c86a690ec601bcc76308c809d8d61edb8 v1.26.10: 1ddcb47ee4f7171736dbacc046a7ceae55411ee09920435c3821b530f4650428 v1.26.9: 14c87cbb9a3fa02308a9546aad192ce2d93e5d1d0296d28ba449079e6a1cb2b2 @@ -345,11 +385,14 @@ kubeadm_checksums: v1.26.1: db101c4bb8e33bd69241de227ed317feee6d44dbd674891e1b9e11c6e8b369bb v1.26.0: 652844c9518786273e094825b74a1988c871552dc6ccf71366558e67409859d1 amd64: + v1.29.0: 629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c + v1.28.5: 2b54078c5ea9e85b27f162f508e0bf834a2753e52a57e896812ec3dca92fe9cd v1.28.4: b4d2531b7cddf782f59555436bc098485b5fa6c05afccdeecf0d62d21d84f5bd v1.28.3: ce3848b1dfa562e0fa2f911a3d8e3bb07ba040eea76654d68e213315c8846ac0 v1.28.2: 6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00 v1.28.1: 6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736 v1.28.0: 12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb + v1.27.9: 78dddac376fa2f04116022cb44ed39ccb9cb0104e05c5b21b220d5151e5c0f86 v1.27.8: f8864769b8b2d7a14f53eb983f23317ff14d68ab76aba71e9de17ce84c38d4eb v1.27.7: bc589219a003b3b94c114e4bcf20549a02657a0c6e5c73f588b37817148892d2 v1.27.6: 2bcdd68957ec25d0689bb56f32b4ec86e38463d2691d5ea21cd109c7afa3aa7c @@ -359,6 +402,7 @@ kubeadm_checksums: v1.27.2: 95c4bfb7929900506a42de4d92280f06efe6b47e0a32cbc1f5a1ed737592977a v1.27.1: c7d32d698e99b90f877025104cb4a9f3f8c707e99e6817940f260135b6d1ad0a v1.27.0: 78d0e04705a7bdb76a514d60f60c073b16334b15f57ee87f064354ca8a233e80 + v1.26.12: 5a5d65acefb50010859be8ffba8e6e059d552ae357e3101c12c62e747a9416a2 v1.26.11: 58f886e39e517ba1a92493f136e80f1b6ea9362966ad9d2accdf2133004161f2 v1.26.10: 27ed1d857f4a315f3d059168c6e25fdbf0559f9c8e59bab6c50e7921f74dadbf v1.26.9: 73e128821dd1f799a75c922218d12f6c4618b8e29cc7dae2a7390fb80092d3d9 @@ -372,11 +416,14 @@ kubeadm_checksums: v1.26.1: 1531abfe96e2e9d8af9219192c65d04df8507a46a081ae1e101478e95d2b63da v1.26.0: 72631449f26b7203701a1b99f6914f31859583a0e247c3ac0f6aaf59ca80af19 ppc64le: + v1.29.0: 4c414a463ed4277e9062c797d1c0435aa7aec2fd1688c5d34e3161c898113cb5 + v1.28.5: a9bf8b18711639d9d002f63cebc22c8df1627737891c640f2229461d19b8c321 v1.28.4: 24e4b42b1d0ec68fc291fcc57fa88ec34b9e8ba758e01639873ef2068222af4a v1.28.3: 0ae62912b057f3228dd7a9fbe2492c4b8c3a661f27a1d46e70b0b6627ccf60fb v1.28.2: fdc28482a4316c84d61b0997c29c4d4c7b11459af9c654fdee3b4a3031f0fcb7 v1.28.1: 73e06f2b614ed5665951f7c059e225a7b0b31319c64a3f57e146fbe7a77fe54e v1.28.0: 146fe9194486e46accd5054fa93939f9608fdbeefefc4bc68e4c40fb4a84ccc9 + v1.27.9: 92da9084fa9f8b8b55436b61ec3c697ef951b0b0416a3b3a7f0dd0e5e4d8cd88 v1.27.8: d65b972cd661cb28972f0df731f9e5b65d959920275bad5ef44ff94d3bb8331d v1.27.7: 6a95a8fb5560a6698d895b2a809485ace0cf5b8c6fb89c843ab20cf89d8b11ad v1.27.6: f2b53fdcd0a71390e84d16facbcd7a581f1309cb8bd0501f9508ebefe5a3498c @@ -386,6 +433,7 @@ kubeadm_checksums: v1.27.2: 412bccd310f4976201d359f0637745944944c0fb2ace315e5e07b180445530c7 v1.27.1: d4c46dcc3d210b6eae0b8c34b3ece9f24b1bb2697175615c451db717a99430fb v1.27.0: cf2860aef800496fee0d9fd8722bd7d17c6609e32d87ca380127151f2ce02bb0 + v1.26.12: fc942d35f179dc6a3e71dff2ab22fa7a940b2f616f7f933819a3892767cee4f9 v1.26.11: bfa424bccba7c10b4a011ef1cc5b282ac5861e30e3e3897ac8ef1cf8cd95b723 v1.26.10: 5dad57d977923d80f20e1d3b9be5dba806642682a0c1bb511c23543c710761cb v1.26.9: 1cd0e3623b93aa1786dddb73570a841323db35df4eca45004db2046550ca5d12 diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index ddb290f915a..257919ab796 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -16,7 +16,7 @@ kubelet_fail_swap_on: true kubelet_swap_behavior: LimitedSwap ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.28.4 +kube_version: v1.28.5 ## The minimum version working kube_version_min_required: v1.26.0 From 1a86b4cb6d17ab54a70b399b0cb834a2688c3a8c Mon Sep 17 00:00:00 2001 From: Romain <58464216+RomainMou@users.noreply.github.com> Date: Thu, 4 Jan 2024 04:00:47 +0100 Subject: [PATCH 008/103] Fix download retry when get_url has no status_code. (#10613) * Fix download retry when get_url has no status_code. * Fix until clause in download role. --- roles/download/tasks/download_file.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/download/tasks/download_file.yml b/roles/download/tasks/download_file.yml index f7f30804846..98b9487f511 100644 --- a/roles/download/tasks/download_file.yml +++ b/roles/download/tasks/download_file.yml @@ -103,7 +103,7 @@ become: "{{ not download_localhost }}" until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg or - get_url_result.status_code == 304" + get_url_result.status_code | default() == 304" retries: "{{ download_retries }}" delay: "{{ retry_stagger | default(5) }}" environment: "{{ proxy_env }}" From 08c0b34270e05dee56f4c30d091284a3b6cd5770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=97=AD?= Date: Fri, 5 Jan 2024 11:45:10 +0800 Subject: [PATCH 009/103] [cert-manager] upgrade to v1.13.2 (#10616) --- README.md | 2 +- .../templates/cert-manager.crds.yml.j2 | 212 +++++++++++------- .../templates/cert-manager.yml.j2 | 51 ++++- .../defaults/main/download.yml | 2 +- 4 files changed, 181 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index b4a721da139..8362ad5bca3 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ Note: Upstart/SysV init based OS types are not supported. - [weave](https://github.com/weaveworks/weave) v2.8.1 - [kube-vip](https://github.com/kube-vip/kube-vip) v0.5.12 - Application - - [cert-manager](https://github.com/jetstack/cert-manager) v1.12.6 + - [cert-manager](https://github.com/jetstack/cert-manager) v1.13.2 - [coredns](https://github.com/coredns/coredns) v1.10.1 - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.4 - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4 diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 index f74ad87c64f..ec96dbba459 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.crds.yml.j2 @@ -20,7 +20,7 @@ metadata: labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' - app.kubernetes.io/instance: 'cert-manager' + app.kubernetes.io/instance: "cert-manager" # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: @@ -355,7 +355,7 @@ spec: - nameserver properties: nameserver: - description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. type: string tsigAlgorithm: description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' @@ -450,7 +450,7 @@ spec: description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' type: array items: - description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n This API may be extended in the future to support additional kinds of parent resources. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." type: object required: - name @@ -462,7 +462,7 @@ spec: maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: - description: "Kind is kind of the referent. \n Support: Core (Gateway) \n Support: Implementation-specific (Other Resources)" + description: "Kind is kind of the referent. \n There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n Support for other resources is Implementation-Specific." type: string default: Gateway maxLength: 63 @@ -474,19 +474,19 @@ spec: maxLength: 253 minLength: 1 namespace: - description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n Support: Core" + description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core" type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: - description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " type: integer format: int32 maximum: 65535 minimum: 1 sectionName: - description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Note that attaching Routes to Services as Parents is part of experimental Mesh support and is not supported for any other purpose. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" type: string maxLength: 253 minLength: 1 @@ -499,7 +499,10 @@ spec: type: object properties: class: - description: The ingress class to use when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of 'class' or 'name' may be specified. + description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressClassName: + description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. @@ -520,7 +523,7 @@ spec: additionalProperties: type: string name: - description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. + description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. type: string podTemplate: description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. @@ -541,7 +544,7 @@ spec: additionalProperties: type: string spec: - description: PodSpec defines overrides for the HTTP01 challenge solver pod. Only the 'priorityClassName', 'nodeSelector', 'affinity', 'serviceAccountName' and 'tolerations' fields are supported currently. All other fields will be ignored. + description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. type: object properties: affinity: @@ -1016,6 +1019,17 @@ spec: topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string + imagePullSecrets: + description: If specified, the pod's imagePullSecrets + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic nodeSelector: description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' type: object @@ -1143,7 +1157,6 @@ spec: type: object required: - role - - secretRef properties: mountPath: description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. @@ -1163,6 +1176,15 @@ spec: name: description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string + serviceAccountRef: + description: A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. + type: object + required: + - name + properties: + name: + description: Name of the ServiceAccount used to request a token. + type: string tokenSecretRef: description: TokenSecretRef authenticates with Vault by presenting a token. type: object @@ -1261,6 +1283,9 @@ spec: description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. type: object properties: + lastPrivateKeyHash: + description: LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + type: string lastRegisteredEmail: description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer type: string @@ -1611,7 +1636,7 @@ spec: - nameserver properties: nameserver: - description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. type: string tsigAlgorithm: description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' @@ -1706,7 +1731,7 @@ spec: description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' type: array items: - description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n This API may be extended in the future to support additional kinds of parent resources. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." type: object required: - name @@ -1718,7 +1743,7 @@ spec: maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: - description: "Kind is kind of the referent. \n Support: Core (Gateway) \n Support: Implementation-specific (Other Resources)" + description: "Kind is kind of the referent. \n There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n Support for other resources is Implementation-Specific." type: string default: Gateway maxLength: 63 @@ -1730,19 +1755,19 @@ spec: maxLength: 253 minLength: 1 namespace: - description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n Support: Core" + description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core" type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: - description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " type: integer format: int32 maximum: 65535 minimum: 1 sectionName: - description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Note that attaching Routes to Services as Parents is part of experimental Mesh support and is not supported for any other purpose. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" type: string maxLength: 253 minLength: 1 @@ -1755,7 +1780,10 @@ spec: type: object properties: class: - description: The ingress class to use when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of 'class' or 'name' may be specified. + description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressClassName: + description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. @@ -1776,7 +1804,7 @@ spec: additionalProperties: type: string name: - description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. + description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. type: string podTemplate: description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. @@ -1797,7 +1825,7 @@ spec: additionalProperties: type: string spec: - description: PodSpec defines overrides for the HTTP01 challenge solver pod. Only the 'priorityClassName', 'nodeSelector', 'affinity', 'serviceAccountName' and 'tolerations' fields are supported currently. All other fields will be ignored. + description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. type: object properties: affinity: @@ -2272,6 +2300,17 @@ spec: topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string + imagePullSecrets: + description: If specified, the pod's imagePullSecrets + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic nodeSelector: description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' type: object @@ -2425,10 +2464,8 @@ spec: type: date schema: openAPIV3Schema: - description: "A CertificateRequest is used to request a signed certificate from one of the configured issuers. \n All fields within the CertificateRequest's `spec` are immutable after creation. A CertificateRequest will either succeed or fail, as denoted by its `status.state` field. \n A CertificateRequest is a one-shot resource, meaning it represents a single point in time request for a certificate and cannot be re-used." + description: "A CertificateRequest is used to request a signed certificate from one of the configured issuers. \n All fields within the CertificateRequest's `spec` are immutable after creation. A CertificateRequest will either succeed or fail, as denoted by its `Ready` status condition and its `status.failureTime` field. \n A CertificateRequest is a one-shot resource, meaning it represents a single point in time request for a certificate and cannot be re-used." type: object - required: - - spec properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' @@ -2439,14 +2476,14 @@ spec: metadata: type: object spec: - description: Desired state of the CertificateRequest resource. + description: Specification of the desired state of the CertificateRequest resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object required: - issuerRef - request properties: duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. This option may be ignored/overridden by some issuer types. + description: Requested 'duration' (i.e. lifetime) of the Certificate. Note that the issuer may choose to ignore the requested duration, just like any other requested attribute. type: string extra: description: Extra contains extra attributes of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. @@ -2462,10 +2499,10 @@ spec: type: string x-kubernetes-list-type: atomic isCA: - description: IsCA will request to mark the certificate as valid for certificate signing when submitting to the issuer. This will automatically add the `cert sign` usage to the list of `usages`. + description: "Requested basic constraints isCA value. Note that the issuer may choose to ignore the requested isCA value, just like any other requested attribute. \n NOTE: If the CSR in the `Request` field has a BasicConstraints extension, it must have the same isCA value as specified here. \n If true, this will automatically add the `cert sign` usage to the list of requested `usages`." type: boolean issuerRef: - description: IssuerRef is a reference to the issuer for this CertificateRequest. If the `kind` field is not set, or set to `Issuer`, an Issuer resource with the given name in the same namespace as the CertificateRequest will be used. If the `kind` field is set to `ClusterIssuer`, a ClusterIssuer with the provided name will be used. The `name` field in this stanza is required at all times. The group field refers to the API group of the issuer which defaults to `cert-manager.io` if empty. + description: "Reference to the issuer responsible for issuing the certificate. If the issuer is namespace-scoped, it must be in the same namespace as the Certificate. If the issuer is cluster-scoped, it can be used from any namespace. \n The `name` field of the reference must always be specified." type: object required: - name @@ -2480,14 +2517,14 @@ spec: description: Name of the resource being referred to. type: string request: - description: The PEM-encoded x509 certificate signing request to be submitted to the CA for signing. + description: "The PEM-encoded X.509 certificate signing request to be submitted to the issuer for signing. \n If the CSR has a BasicConstraints extension, its isCA attribute must match the `isCA` value of this CertificateRequest. If the CSR has a KeyUsage extension, its key usages must match the key usages in the `usages` field of this CertificateRequest. If the CSR has a ExtKeyUsage extension, its extended key usages must match the extended key usages in the `usages` field of this CertificateRequest." type: string format: byte uid: description: UID contains the uid of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. type: string usages: - description: Usages is the set of x509 usages that are requested for the certificate. If usages are set they SHOULD be encoded inside the CSR spec Defaults to `digital signature` and `key encipherment` if not specified. + description: "Requested key usages and extended key usages. \n NOTE: If the CSR in the `Request` field has uses the KeyUsage or ExtKeyUsage extension, these extensions must have the same values as specified here without any additional values. \n If unset, defaults to `digital signature` and `key encipherment`." type: array items: description: "KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 \n Valid KeyUsage values are as follows: \"signing\", \"digital signature\", \"content commitment\", \"key encipherment\", \"key agreement\", \"data encipherment\", \"cert sign\", \"crl sign\", \"encipher only\", \"decipher only\", \"any\", \"server auth\", \"client auth\", \"code signing\", \"email protection\", \"s/mime\", \"ipsec end system\", \"ipsec tunnel\", \"ipsec user\", \"timestamping\", \"ocsp signing\", \"microsoft sgc\", \"netscape sgc\"" @@ -2520,19 +2557,19 @@ spec: description: Username contains the name of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. type: string status: - description: Status of the CertificateRequest. This is set and managed automatically. + description: 'Status of the CertificateRequest. This is set and managed automatically. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' type: object properties: ca: - description: The PEM encoded x509 certificate of the signer, also known as the CA (Certificate Authority). This is set on a best-effort basis by different issuers. If not set, the CA is assumed to be unknown/not available. + description: The PEM encoded X.509 certificate of the signer, also known as the CA (Certificate Authority). This is set on a best-effort basis by different issuers. If not set, the CA is assumed to be unknown/not available. type: string format: byte certificate: - description: The PEM encoded x509 certificate resulting from the certificate signing request. If not set, the CertificateRequest has either not been completed or has failed. More information on failure can be found by checking the `conditions` field. + description: The PEM encoded X.509 certificate resulting from the certificate signing request. If not set, the CertificateRequest has either not been completed or has failed. More information on failure can be found by checking the `conditions` field. type: string format: byte conditions: - description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready` and `InvalidRequest`. + description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`, `InvalidRequest`, `Approved` and `Denied`. type: array items: description: CertificateRequestCondition contains condition information for a CertificateRequest. @@ -2579,7 +2616,7 @@ metadata: labels: app: 'cert-manager' app.kubernetes.io/name: 'cert-manager' - app.kubernetes.io/instance: 'cert-manager' + app.kubernetes.io/instance: "cert-manager" # Generated labels app.kubernetes.io/version: "{{ cert_manager_version }}" spec: @@ -2914,7 +2951,7 @@ spec: - nameserver properties: nameserver: - description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. type: string tsigAlgorithm: description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' @@ -3009,7 +3046,7 @@ spec: description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' type: array items: - description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n This API may be extended in the future to support additional kinds of parent resources. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." type: object required: - name @@ -3021,7 +3058,7 @@ spec: maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: - description: "Kind is kind of the referent. \n Support: Core (Gateway) \n Support: Implementation-specific (Other Resources)" + description: "Kind is kind of the referent. \n There are two kinds of parent resources with \"Core\" support: \n * Gateway (Gateway conformance profile) * Service (Mesh conformance profile, experimental, ClusterIP Services only) \n Support for other resources is Implementation-Specific." type: string default: Gateway maxLength: 63 @@ -3033,19 +3070,19 @@ spec: maxLength: 253 minLength: 1 namespace: - description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n Support: Core" + description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n ParentRefs from a Route to a Service in the same namespace are \"producer\" routes, which apply default routing rules to inbound connections from any namespace to the Service. \n ParentRefs from a Route to a Service in a different namespace are \"consumer\" routes, and these routing rules are only applied to outbound connections originating from the same namespace as the Route, for which the intended destination of the connections are a Service targeted as a ParentRef of the Route. \n Support: Core" type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: - description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n When the parent resource is a Service, this targets a specific port in the Service spec. When both Port (experimental) and SectionName are specified, the name and port of the selected port must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " type: integer format: int32 maximum: 65535 minimum: 1 sectionName: - description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. * Service: Port Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. Note that attaching Routes to Services as Parents is part of experimental Mesh support and is not supported for any other purpose. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" type: string maxLength: 253 minLength: 1 @@ -3058,7 +3095,10 @@ spec: type: object properties: class: - description: The ingress class to use when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of 'class' or 'name' may be specified. + description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + type: string + ingressClassName: + description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. @@ -3079,7 +3119,7 @@ spec: additionalProperties: type: string name: - description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. + description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. type: string podTemplate: description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. @@ -3100,7 +3140,7 @@ spec: additionalProperties: type: string spec: - description: PodSpec defines overrides for the HTTP01 challenge solver pod. Only the 'priorityClassName', 'nodeSelector', 'affinity', 'serviceAccountName' and 'tolerations' fields are supported currently. All other fields will be ignored. + description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. type: object properties: affinity: @@ -3575,6 +3615,17 @@ spec: topologyKey: description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. type: string + imagePullSecrets: + description: If specified, the pod's imagePullSecrets + type: array + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + type: object + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + x-kubernetes-map-type: atomic nodeSelector: description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' type: object @@ -3702,7 +3753,6 @@ spec: type: object required: - role - - secretRef properties: mountPath: description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. @@ -3722,6 +3772,15 @@ spec: name: description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string + serviceAccountRef: + description: A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. + type: object + required: + - name + properties: + name: + description: Name of the ServiceAccount used to request a token. + type: string tokenSecretRef: description: TokenSecretRef authenticates with Vault by presenting a token. type: object @@ -3820,6 +3879,9 @@ spec: description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. type: object properties: + lastPrivateKeyHash: + description: LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + type: string lastRegisteredEmail: description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer type: string @@ -3915,10 +3977,8 @@ spec: type: date schema: openAPIV3Schema: - description: "A Certificate resource should be created to ensure an up to date and signed x509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`. \n The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`)." + description: "A Certificate resource should be created to ensure an up to date and signed X.509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`. \n The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`)." type: object - required: - - spec properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' @@ -3929,14 +3989,14 @@ spec: metadata: type: object spec: - description: Desired state of the Certificate resource. + description: Specification of the desired state of the Certificate resource. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object required: - issuerRef - secretName properties: additionalOutputFormats: - description: AdditionalOutputFormats defines extra output formats of the private key and signed certificate chain to be written to this Certificate's target Secret. This is an Alpha Feature and is only enabled with the `--feature-gates=AdditionalCertificateOutputFormats=true` option on both the controller and webhook components. + description: "Defines extra output formats of the private key and signed certificate chain to be written to this Certificate's target Secret. \n This is an Alpha Feature and is only enabled with the `--feature-gates=AdditionalCertificateOutputFormats=true` option set on both the controller and webhook components." type: array items: description: CertificateAdditionalOutputFormat defines an additional output format of a Certificate resource. These contain supplementary data formats of the signed certificate chain and paired private key. @@ -3951,34 +4011,34 @@ spec: - DER - CombinedPEM commonName: - description: 'CommonName is a common name to be used on the Certificate. The CommonName should have a length of 64 characters or fewer to avoid generating invalid CSRs. This value is ignored by TLS clients when any subject alt name is set. This is x509 behaviour: https://tools.ietf.org/html/rfc6125#section-6.4.4' + description: "Requested common name X509 certificate subject attribute. More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 NOTE: TLS clients will ignore this value when any subject alternative name is set (see https://tools.ietf.org/html/rfc6125#section-6.4.4). \n Should have a length of 64 characters or fewer to avoid generating invalid CSRs. Cannot be set if the `literalSubject` field is set." type: string dnsNames: - description: DNSNames is a list of DNS subjectAltNames to be set on the Certificate. + description: Requested DNS subject alternative names. type: array items: type: string duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. This option may be ignored/overridden by some issuer types. If unset this defaults to 90 days. Certificate will be renewed either 2/3 through its duration or `renewBefore` period before its expiry, whichever is later. Minimum accepted duration is 1 hour. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration + description: "Requested 'duration' (i.e. lifetime) of the Certificate. Note that the issuer may choose to ignore the requested duration, just like any other requested attribute. \n If unset, this defaults to 90 days. Minimum accepted duration is 1 hour. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration." type: string emailAddresses: - description: EmailAddresses is a list of email subjectAltNames to be set on the Certificate. + description: Requested email subject alternative names. type: array items: type: string encodeUsagesInRequest: - description: EncodeUsagesInRequest controls whether key usages should be present in the CertificateRequest + description: "Whether the KeyUsage and ExtKeyUsage extensions should be set in the encoded CSR. \n This option defaults to true, and should only be disabled if the target issuer does not support CSRs with these X509 KeyUsage/ ExtKeyUsage extensions." type: boolean ipAddresses: - description: IPAddresses is a list of IP address subjectAltNames to be set on the Certificate. + description: Requested IP address subject alternative names. type: array items: type: string isCA: - description: IsCA will mark this Certificate as valid for certificate signing. This will automatically add the `cert sign` usage to the list of `usages`. + description: "Requested basic constraints isCA value. The isCA value is used to set the `isCA` field on the created CertificateRequest resources. Note that the issuer may choose to ignore the requested isCA value, just like any other requested attribute. \n If true, this will automatically add the `cert sign` usage to the list of requested `usages`." type: boolean issuerRef: - description: IssuerRef is a reference to the issuer for this certificate. If the `kind` field is not set, or set to `Issuer`, an Issuer resource with the given name in the same namespace as the Certificate will be used. If the `kind` field is set to `ClusterIssuer`, a ClusterIssuer with the provided name will be used. The `name` field in this stanza is required at all times. + description: "Reference to the issuer responsible for issuing the certificate. If the issuer is namespace-scoped, it must be in the same namespace as the Certificate. If the issuer is cluster-scoped, it can be used from any namespace. \n The `name` field of the reference must always be specified." type: object required: - name @@ -3993,7 +4053,7 @@ spec: description: Name of the resource being referred to. type: string keystores: - description: Keystores configures additional keystore output formats stored in the `secretName` Secret resource. + description: Additional keystore output formats to be stored in the Certificate's Secret. type: object properties: jks: @@ -4004,7 +4064,7 @@ spec: - passwordSecretRef properties: create: - description: Create enables JKS keystore creation for the Certificate. If true, a file named `keystore.jks` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. A file named `truststore.jks` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority + description: Create enables JKS keystore creation for the Certificate. If true, a file named `keystore.jks` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.jks` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority type: boolean passwordSecretRef: description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the JKS keystore. @@ -4026,7 +4086,7 @@ spec: - passwordSecretRef properties: create: - description: Create enables PKCS12 keystore creation for the Certificate. If true, a file named `keystore.p12` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. A file named `truststore.p12` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority + description: Create enables PKCS12 keystore creation for the Certificate. If true, a file named `keystore.p12` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.p12` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority type: boolean passwordSecretRef: description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the PKCS12 keystore. @@ -4041,46 +4101,46 @@ spec: description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string literalSubject: - description: LiteralSubject is an LDAP formatted string that represents the [X.509 Subject field](https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6). Use this *instead* of the Subject field if you need to ensure the correct ordering of the RDN sequence, such as when issuing certs for LDAP authentication. See https://github.com/cert-manager/cert-manager/issues/3203, https://github.com/cert-manager/cert-manager/issues/4424. This field is alpha level and is only supported by cert-manager installations where LiteralCertificateSubject feature gate is enabled on both cert-manager controller and webhook. + description: "Requested X.509 certificate subject, represented using the LDAP \"String Representation of a Distinguished Name\" [1]. Important: the LDAP string format also specifies the order of the attributes in the subject, this is important when issuing certs for LDAP authentication. Example: `CN=foo,DC=corp,DC=example,DC=com` More info [1]: https://datatracker.ietf.org/doc/html/rfc4514 More info: https://github.com/cert-manager/cert-manager/issues/3203 More info: https://github.com/cert-manager/cert-manager/issues/4424 \n Cannot be set if the `subject` or `commonName` field is set. This is an Alpha Feature and is only enabled with the `--feature-gates=LiteralCertificateSubject=true` option set on both the controller and webhook components." type: string privateKey: - description: Options to control private keys used for the Certificate. + description: Private key options. These include the key algorithm and size, the used encoding and the rotation policy. type: object properties: algorithm: - description: Algorithm is the private key algorithm of the corresponding private key for this certificate. If provided, allowed values are either `RSA`,`Ed25519` or `ECDSA` If `algorithm` is specified and `size` is not provided, key size of 256 will be used for `ECDSA` key algorithm and key size of 2048 will be used for `RSA` key algorithm. key size is ignored when using the `Ed25519` key algorithm. + description: "Algorithm is the private key algorithm of the corresponding private key for this certificate. \n If provided, allowed values are either `RSA`, `ECDSA` or `Ed25519`. If `algorithm` is specified and `size` is not provided, key size of 2048 will be used for `RSA` key algorithm and key size of 256 will be used for `ECDSA` key algorithm. key size is ignored when using the `Ed25519` key algorithm." type: string enum: - RSA - ECDSA - Ed25519 encoding: - description: The private key cryptography standards (PKCS) encoding for this certificate's private key to be encoded in. If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1 and PKCS#8, respectively. Defaults to `PKCS1` if not specified. + description: "The private key cryptography standards (PKCS) encoding for this certificate's private key to be encoded in. \n If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1 and PKCS#8, respectively. Defaults to `PKCS1` if not specified." type: string enum: - PKCS1 - PKCS8 rotationPolicy: - description: RotationPolicy controls how private keys should be regenerated when a re-issuance is being processed. If set to Never, a private key will only be generated if one does not already exist in the target `spec.secretName`. If one does exists but it does not have the correct algorithm or size, a warning will be raised to await user intervention. If set to Always, a private key matching the specified requirements will be generated whenever a re-issuance occurs. Default is 'Never' for backward compatibility. + description: "RotationPolicy controls how private keys should be regenerated when a re-issuance is being processed. \n If set to `Never`, a private key will only be generated if one does not already exist in the target `spec.secretName`. If one does exists but it does not have the correct algorithm or size, a warning will be raised to await user intervention. If set to `Always`, a private key matching the specified requirements will be generated whenever a re-issuance occurs. Default is `Never` for backward compatibility." type: string enum: - Never - Always size: - description: Size is the key bit size of the corresponding private key for this certificate. If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`, and will default to `2048` if not specified. If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`, and will default to `256` if not specified. If `algorithm` is set to `Ed25519`, Size is ignored. No other values are allowed. + description: "Size is the key bit size of the corresponding private key for this certificate. \n If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`, and will default to `2048` if not specified. If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`, and will default to `256` if not specified. If `algorithm` is set to `Ed25519`, Size is ignored. No other values are allowed." type: integer renewBefore: - description: How long before the currently issued certificate's expiry cert-manager should renew the certificate. The default is 2/3 of the issued certificate's duration. Minimum accepted value is 5 minutes. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration + description: "How long before the currently issued certificate's expiry cert-manager should renew the certificate. For example, if a certificate is valid for 60 minutes, and `renewBefore=10m`, cert-manager will begin to attempt to renew the certificate 50 minutes after it was issued (i.e. when there are 10 minutes remaining until the certificate is no longer valid). \n NOTE: The actual lifetime of the issued certificate is used to determine the renewal time. If an issuer returns a certificate with a different lifetime than the one requested, cert-manager will use the lifetime of the issued certificate. \n If unset, this defaults to 1/3 of the issued certificate's lifetime. Minimum accepted value is 5 minutes. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration." type: string revisionHistoryLimit: - description: revisionHistoryLimit is the maximum number of CertificateRequest revisions that are maintained in the Certificate's history. Each revision represents a single `CertificateRequest` created by this Certificate, either when it was created, renewed, or Spec was changed. Revisions will be removed by oldest first if the number of revisions exceeds this number. If set, revisionHistoryLimit must be a value of `1` or greater. If unset (`nil`), revisions will not be garbage collected. Default value is `nil`. + description: "The maximum number of CertificateRequest revisions that are maintained in the Certificate's history. Each revision represents a single `CertificateRequest` created by this Certificate, either when it was created, renewed, or Spec was changed. Revisions will be removed by oldest first if the number of revisions exceeds this number. \n If set, revisionHistoryLimit must be a value of `1` or greater. If unset (`nil`), revisions will not be garbage collected. Default value is `nil`." type: integer format: int32 secretName: - description: SecretName is the name of the secret resource that will be automatically created and managed by this Certificate resource. It will be populated with a private key and certificate, signed by the denoted issuer. + description: Name of the Secret resource that will be automatically created and managed by this Certificate resource. It will be populated with a private key and certificate, signed by the denoted issuer. The Secret resource lives in the same namespace as the Certificate resource. type: string secretTemplate: - description: SecretTemplate defines annotations and labels to be copied to the Certificate's Secret. Labels and annotations on the Secret will be changed as they appear on the SecretTemplate when added or removed. SecretTemplate annotations are added in conjunction with, and cannot overwrite, the base set of annotations cert-manager sets on the Certificate's Secret. + description: Defines annotations and labels to be copied to the Certificate's Secret. Labels and annotations on the Secret will be changed as they appear on the SecretTemplate when added or removed. SecretTemplate annotations are added in conjunction with, and cannot overwrite, the base set of annotations cert-manager sets on the Certificate's Secret. type: object properties: annotations: @@ -4094,7 +4154,7 @@ spec: additionalProperties: type: string subject: - description: Full X509 name specification (https://golang.org/pkg/crypto/x509/pkix/#Name). + description: "Requested set of X509 certificate subject attributes. More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 \n The common name attribute is specified separately in the `commonName` field. Cannot be set if the `literalSubject` field is set." type: object properties: countries: @@ -4136,12 +4196,12 @@ spec: items: type: string uris: - description: URIs is a list of URI subjectAltNames to be set on the Certificate. + description: Requested URI subject alternative names. type: array items: type: string usages: - description: Usages is the set of x509 usages that are requested for the certificate. Defaults to `digital signature` and `key encipherment` if not specified. + description: "Requested key usages and extended key usages. These usages are used to set the `usages` field on the created CertificateRequest resources. If `encodeUsagesInRequest` is unset or set to `true`, the usages will additionally be encoded in the `request` field which contains the CSR blob. \n If unset, defaults to `digital signature` and `key encipherment`." type: array items: description: "KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 \n Valid KeyUsage values are as follows: \"signing\", \"digital signature\", \"content commitment\", \"key encipherment\", \"key agreement\", \"data encipherment\", \"cert sign\", \"crl sign\", \"encipher only\", \"decipher only\", \"any\", \"server auth\", \"client auth\", \"code signing\", \"email protection\", \"s/mime\", \"ipsec end system\", \"ipsec tunnel\", \"ipsec user\", \"timestamping\", \"ocsp signing\", \"microsoft sgc\", \"netscape sgc\"" @@ -4171,7 +4231,7 @@ spec: - microsoft sgc - netscape sgc status: - description: Status of the Certificate. This is set and managed automatically. + description: 'Status of the Certificate. This is set and managed automatically. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' type: object properties: conditions: @@ -4215,7 +4275,7 @@ spec: description: The number of continuous failed issuance attempts up till now. This field gets removed (if set) on a successful issuance and gets set to 1 if unset and an issuance has failed. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). type: integer lastFailureTime: - description: LastFailureTime is the time as recorded by the Certificate controller of the most recent failure to complete a CertificateRequest for this Certificate resource. If set, cert-manager will not re-request another Certificate until 1 hour has elapsed from this time. + description: LastFailureTime is set only if the lastest issuance for this Certificate failed and contains the time of the failure. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). If the latest issuance has succeeded this field will be unset. type: string format: date-time nextPrivateKeySecretName: @@ -4226,7 +4286,7 @@ spec: type: string format: date-time notBefore: - description: The time after which the certificate stored in the secret named by this resource in spec.secretName is valid. + description: The time after which the certificate stored in the secret named by this resource in `spec.secretName` is valid. type: string format: date-time renewalTime: diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 index d4e791c3800..2bbda1108d8 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 @@ -60,6 +60,20 @@ metadata: app.kubernetes.io/component: "webhook" app.kubernetes.io/version: "{{ cert_manager_version }}" --- +# Source: cert-manager/deploy/charts/cert-manager/templates/controller-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cert-manager + namespace: {{ cert_manager_namespace }} + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" +data: +--- # Source: cert-manager/deploy/charts/cert-manager/templates/webhook-config.yaml apiVersion: v1 kind: ConfigMap @@ -71,6 +85,7 @@ metadata: app.kubernetes.io/name: webhook app.kubernetes.io/instance: cert-manager app.kubernetes.io/component: "webhook" + app.kubernetes.io/version: "{{ cert_manager_version }}" data: --- # Source: cert-manager/deploy/charts/cert-manager/templates/cainjector-rbac.yaml @@ -96,13 +111,13 @@ rules: verbs: ["get", "create", "update", "patch"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["apiregistration.k8s.io"] resources: ["apiservices"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "patch"] --- # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml # Issuer controller role @@ -330,6 +345,23 @@ rules: # Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: cert-manager-cluster-view + labels: + app: cert-manager + app.kubernetes.io/name: cert-manager + app.kubernetes.io/instance: cert-manager + app.kubernetes.io/component: "controller" + app.kubernetes.io/version: "{{ cert_manager_version }}" + rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["clusterissuers"] + verbs: ["get", "list", "watch"] +--- +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: cert-manager-view labels: @@ -341,6 +373,7 @@ metadata: rbac.authorization.k8s.io/aggregate-to-view: "true" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" rules: - apiGroups: ["cert-manager.io"] resources: ["certificates", "certificaterequests", "issuers"] @@ -476,7 +509,7 @@ subjects: namespace: {{ cert_manager_namespace }} kind: ServiceAccount --- -# Source: cert-manager/deploy/charts/cert-manager/templates//rbac.yaml +# Source: cert-manager/deploy/charts/cert-manager/templates/rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -854,6 +887,7 @@ spec: app.kubernetes.io/version: "{{ cert_manager_version }}" spec: serviceAccountName: cert-manager-cainjector + enableServiceLinks: false securityContext: runAsNonRoot: true seccompProfile: @@ -947,6 +981,7 @@ spec: prometheus.io/port: '9402' spec: serviceAccountName: cert-manager + enableServiceLinks: false securityContext: runAsNonRoot: true seccompProfile: @@ -966,6 +1001,9 @@ spec: - containerPort: 9402 name: http-metrics protocol: TCP + - containerPort: 9403 + name: http-healthz + protocol: TCP securityContext: allowPrivilegeEscalation: false capabilities: @@ -1051,6 +1089,7 @@ spec: app.kubernetes.io/version: "{{ cert_manager_version }}" spec: serviceAccountName: cert-manager-webhook + enableServiceLinks: false securityContext: runAsNonRoot: true seccompProfile: @@ -1194,10 +1233,6 @@ webhooks: operator: "NotIn" values: - "true" - - key: "name" - operator: "NotIn" - values: - - cert-manager rules: - apiGroups: - "cert-manager.io" diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index d30c6efe399..675e200b108 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -314,7 +314,7 @@ ingress_nginx_kube_webhook_certgen_image_repo: "{{ kube_image_repo }}/ingress-ng ingress_nginx_kube_webhook_certgen_image_tag: "v20231011-8b53cabe0" alb_ingress_image_repo: "{{ docker_image_repo }}/amazon/aws-alb-ingress-controller" alb_ingress_image_tag: "v1.1.9" -cert_manager_version: "v1.12.6" +cert_manager_version: "v1.13.2" cert_manager_controller_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-controller" cert_manager_controller_image_tag: "{{ cert_manager_version }}" cert_manager_cainjector_image_repo: "{{ quay_image_repo }}/jetstack/cert-manager-cainjector" From eda7ea56951a2e5f12f466d009516f70c549886d Mon Sep 17 00:00:00 2001 From: Ryan Lonergan Date: Sun, 7 Jan 2024 20:43:02 -0500 Subject: [PATCH 010/103] feat: add support for Cilium 1.14 (#10684) * update cilium configmap template for new routing mode and tunnel-protocol options Ryan Lonergan ryan.tlonergan@gmail.com * add rbac for new cilium crd in 1.14 Ryan Lonergan ryan.tlonergan@gmail.com * add conditional for cni-install.sh that's no longer included in cilium 1.14 Ryan Lonergan ryan.tlonergan@gmail.com * Update roles/network_plugin/cilium/templates/cilium/ds.yml.j2 Co-authored-by: Cyclinder --------- Co-authored-by: Cyclinder --- .../network_plugin/cilium/templates/cilium/config.yml.j2 | 7 +++++++ roles/network_plugin/cilium/templates/cilium/cr.yml.j2 | 9 +++++++++ roles/network_plugin/cilium/templates/cilium/ds.yml.j2 | 2 ++ 3 files changed, 18 insertions(+) diff --git a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 b/roles/network_plugin/cilium/templates/cilium/config.yml.j2 index 399d8ced87e..9fb5f281953 100644 --- a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/config.yml.j2 @@ -115,7 +115,14 @@ data: # - disabled # - vxlan (default) # - geneve +{% if cilium_version | regex_replace('v') is version('1.14.0', '<') %} tunnel: "{{ cilium_tunnel_mode }}" +{% elif cilium_version | regex_replace('v') is version('1.14.0', '>=') and cilium_tunnel_mode == 'disabled' %} + routing-mode: 'native' +{% elif cilium_version | regex_replace('v') is version('1.14.0', '>=') and cilium_tunnel_mode != 'disabled' %} + routing-mode: 'tunnel' + tunnel-protocol: "{{ cilium_tunnel_mode }}" +{% endif %} # Enable Bandwidth Manager # Cilium’s bandwidth manager supports the kubernetes.io/egress-bandwidth Pod annotation. diff --git a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 index a16211c1792..4ce747f0f52 100644 --- a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 @@ -120,3 +120,12 @@ rules: - list - watch {% endif %} +{% if cilium_version | regex_replace('v') is version('1.14', '>=') %} +- apiGroups: + - cilium.io + resources: + - ciliumcidrgroups + verbs: + - list + - watch +{% endif %} diff --git a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 index 38360342b3b..658f53bfeff 100644 --- a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 @@ -106,6 +106,7 @@ spec: - {{ env_var | to_nice_yaml(indent=2) | indent(10) }} {% endfor %} lifecycle: +{% if cilium_version | regex_replace('v') is version('1.14', '<') %} postStart: exec: command: @@ -114,6 +115,7 @@ spec: {% if cilium_version | regex_replace('v') is version('1.12', '>=') %} - "--enable-debug={{ cilium_debug | string | lower }}" - "--log-file={{ cilium_cni_log_file }}" +{% endif %} {% endif %} preStop: exec: From ddf5c6ee123dec680b4aa348ba74027e204a9568 Mon Sep 17 00:00:00 2001 From: Louis Tu <92532497+tu1h@users.noreply.github.com> Date: Tue, 9 Jan 2024 00:38:05 +0800 Subject: [PATCH 011/103] Update coredns rolling update strategy (#10748) Signed-off-by: tu1h --- .../kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 index 466e4463bed..cbdca572759 100644 --- a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 @@ -12,7 +12,7 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxUnavailable: 0 + maxUnavailable: 1 maxSurge: 10% selector: matchLabels: From 2e8b72e27843e9cf70e9c8fe048e23c80ef39976 Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Tue, 9 Jan 2024 00:38:14 +0800 Subject: [PATCH 012/103] fix disable swap in centos (#10751) --- .../preinstall/tasks/0010-swapoff.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/roles/kubernetes/preinstall/tasks/0010-swapoff.yml b/roles/kubernetes/preinstall/tasks/0010-swapoff.yml index 53bcf2d1218..45474c844d4 100644 --- a/roles/kubernetes/preinstall/tasks/0010-swapoff.yml +++ b/roles/kubernetes/preinstall/tasks/0010-swapoff.yml @@ -1,4 +1,22 @@ --- +- name: Check if /etc/fstab exists + stat: + path: "/etc/fstab" + get_attributes: no + get_checksum: no + get_mime: no + register: fstab_file + +- name: Remove swapfile from /etc/fstab + ansible.posix.mount: + name: "{{ item }}" + fstype: swap + state: absent + loop: + - swap + - none + when: fstab_file.stat.exists + - name: Mask swap.target (persist swapoff) ansible.builtin.systemd_service: name: swap.target From a656b7ed9af78a37520cceb30af8fec0aae1fa88 Mon Sep 17 00:00:00 2001 From: Louis Tu <92532497+tu1h@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:22:13 +0800 Subject: [PATCH 013/103] Add kube_vip_lb_fwdmethod option for kube-vip (#10762) Signed-off-by: tu1h --- docs/kube-vip.md | 6 ++++++ roles/kubernetes/node/defaults/main.yml | 1 + .../node/templates/manifests/kube-vip.manifest.j2 | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/docs/kube-vip.md b/docs/kube-vip.md index 846ec099a11..d5878ab6af0 100644 --- a/docs/kube-vip.md +++ b/docs/kube-vip.md @@ -70,3 +70,9 @@ If using [control plane load-balancing](https://kube-vip.io/docs/about/architect ```yaml kube_vip_lb_enable: true ``` + +In addition, [load-balancing method](https://kube-vip.io/docs/installation/flags/#environment-variables) could be changed: + +```yaml +kube_vip_lb_fwdmethod: masquerade +``` diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index cbe95835cfb..0522187b95a 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -86,6 +86,7 @@ kube_vip_bgppeers: kube_vip_address: kube_vip_enableServicesElection: false kube_vip_lb_enable: false +kube_vip_lb_fwdmethod: local # Requests for load balancer app loadbalancer_apiserver_memory_requests: 32M diff --git a/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 index b95b4746967..936ad44d267 100644 --- a/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 @@ -77,6 +77,10 @@ spec: {% if kube_vip_lb_enable %} - name: lb_enable value: "true" +{% endif %} +{% if kube_vip_lb_fwdmethod %} + - name: lb_fwdmethod + value: {{ kube_vip_lb_fwdmethod | string | to_json }} {% endif %} image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} From 8c09c3fda2367095f26f46e8a537851e5e6a5fed Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Tue, 9 Jan 2024 17:20:16 +0800 Subject: [PATCH 014/103] fix image pull in insecure-registry (#10775) --- roles/kubespray-defaults/defaults/main/download.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 675e200b108..5feb32d9e8d 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -58,7 +58,7 @@ docker_image_pull_command: "{{ docker_bin_dir }}/docker pull" docker_image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs -i {{ '{{' }} docker_bin_dir }}/docker inspect -f {% raw %}'{{ '{{' }} if .RepoTags }}{{ '{{' }} join .RepoTags \",\" }}{{ '{{' }} end }}{{ '{{' }} if .RepoDigests }},{{ '{{' }} join .RepoDigests \",\" }}{{ '{{' }} end }}' {% endraw %} {} | tr '\n' ','" nerdctl_image_info_command: "{{ bin_dir }}/nerdctl -n k8s.io images --format '{% raw %}{{ .Repository }}:{{ .Tag }}{% endraw %}' 2>/dev/null | grep -v ^:$ | tr '\n' ','" # Using the ctr instead of nerdctl to workdaround the https://github.com/kubernetes-sigs/kubespray/issues/10670 -nerdctl_image_pull_command: "{{ bin_dir }}/ctr -n k8s.io images pull" +nerdctl_image_pull_command: "{{ bin_dir }}/ctr -n k8s.io images pull{% if containerd_registries_mirrors is defined %} --hosts-dir {{ containerd_cfg_dir }}/certs.d{%- endif -%}" crictl_image_info_command: "{{ bin_dir }}/crictl images --verbose | awk -F ': ' '/RepoTags|RepoDigests/ {print $2}' | tr '\n' ','" crictl_image_pull_command: "{{ bin_dir }}/crictl pull" From a306f15a741e69a4706db534149424053d878b6d Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Thu, 11 Jan 2024 18:23:15 +0800 Subject: [PATCH 015/103] bump vagrant 2.3.7 (#10787) --- .gitlab-ci/lint.yml | 2 +- Vagrantfile | 1 + pipeline.Dockerfile | 2 +- test-infra/vagrant-docker/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci/lint.yml b/.gitlab-ci/lint.yml index d8cebd22737..5e4b81abb0b 100644 --- a/.gitlab-ci/lint.yml +++ b/.gitlab-ci/lint.yml @@ -14,7 +14,7 @@ vagrant-validate: stage: unit-tests tags: [light] variables: - VAGRANT_VERSION: 2.3.4 + VAGRANT_VERSION: 2.3.7 script: - ./tests/scripts/vagrant-validate.sh except: ['triggers', 'master'] diff --git a/Vagrantfile b/Vagrantfile index 7ba30f32e57..49873766872 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -263,6 +263,7 @@ Vagrant.configure("2") do |config| if i == $num_instances node.vm.provision "ansible" do |ansible| ansible.playbook = $playbook + ansible.compatibility_mode = "2.0" ansible.verbose = $ansible_verbosity $ansible_inventory_path = File.join( $inventory, "hosts.ini") if File.exist?($ansible_inventory_path) diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile index 4eba6a1a550..cafcfcf149e 100644 --- a/pipeline.Dockerfile +++ b/pipeline.Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:jammy-20230308 # Pip needs this as well at the moment to install ansible # (and potentially other packages) # See: https://github.com/pypa/pip/issues/10219 -ENV VAGRANT_VERSION=2.3.4 \ +ENV VAGRANT_VERSION=2.3.7 \ VAGRANT_DEFAULT_PROVIDER=libvirt \ VAGRANT_ANSIBLE_TAGS=facts \ LANG=C.UTF-8 \ diff --git a/test-infra/vagrant-docker/Dockerfile b/test-infra/vagrant-docker/Dockerfile index f12a7409ae8..7a0f0e08da4 100644 --- a/test-infra/vagrant-docker/Dockerfile +++ b/test-infra/vagrant-docker/Dockerfile @@ -3,7 +3,7 @@ ARG KUBESPRAY_VERSION FROM quay.io/kubespray/kubespray:${KUBESPRAY_VERSION} -ENV VAGRANT_VERSION=2.3.4 +ENV VAGRANT_VERSION=2.3.7 ENV VAGRANT_DEFAULT_PROVIDER=libvirt ENV VAGRANT_ANSIBLE_TAGS=facts From bb67d9524d940d4462342386a0d9f3ac075bd82e Mon Sep 17 00:00:00 2001 From: Serge Hartmann Date: Thu, 11 Jan 2024 11:49:35 +0100 Subject: [PATCH 016/103] Fix crio_version version comparison (#10780) Signed-off-by: serge Hartmann --- roles/container-engine/cri-o/templates/crio.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index 81d5a421e0b..6f9b84f14eb 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -382,7 +382,7 @@ enable_metrics = {{ crio_enable_metrics | bool | lower }} # The port on which the metrics server will listen. metrics_port = {{ crio_metrics_port }} -{% if nri_enabled and crio_version >= v1.26.0 %} +{% if nri_enabled and crio_version is version('v1.26.0', operator='>=') %} [crio.nri] enable_nri=true From e90cae9344c1de85ddae90f814374d4f6e7f1536 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 11 Jan 2024 15:49:31 +0100 Subject: [PATCH 017/103] Refactor check_galaxy + fix version (#10729) * Remove checks for docs using exact tags Instead use a more generic documentation for installing kubespray as a collection from git. * Check that we upgraded galaxy.yml to next version This is only intented to check for human error. The version in galaxy should be the next (which does not mean the same if we're on master or a release branch). * Set collection version to KUBESPRAY_NEXT_VERSION --- docs/ansible_collection.md | 2 +- galaxy.yml | 2 +- tests/scripts/check_galaxy_version.sh | 14 ++++---------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/docs/ansible_collection.md b/docs/ansible_collection.md index cbe1ad2c5e5..bcf23baa391 100644 --- a/docs/ansible_collection.md +++ b/docs/ansible_collection.md @@ -15,7 +15,7 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a collections: - name: https://github.com/kubernetes-sigs/kubespray type: git - version: v2.23.1 + version: master # use the appropriate tag or branch for the version you need ``` 2. Install your collection diff --git a/galaxy.yml b/galaxy.yml index 7b487ca50dc..63731ca141a 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,7 +2,7 @@ namespace: kubernetes_sigs description: Deploy a production ready Kubernetes cluster name: kubespray -version: 2.23.1 +version: 2.24.0 readme: README.md authors: - luksi1 diff --git a/tests/scripts/check_galaxy_version.sh b/tests/scripts/check_galaxy_version.sh index b6679dba472..d663f121f07 100755 --- a/tests/scripts/check_galaxy_version.sh +++ b/tests/scripts/check_galaxy_version.sh @@ -2,17 +2,11 @@ set -e version_from_galaxy=$(grep "^version:" galaxy.yml | awk '{print $2}') -version_from_docs=$(grep -P "^\s+version:\sv\d+\.\d+\.\d+" docs/ansible_collection.md | awk '{print $2}') -if [[ $KUBESPRAY_VERSION != "v${version_from_galaxy}" ]] +# TODO: compute the next expected version somehow +if [[ $KUBESPRAY_VERSION == "v${version_from_galaxy}" ]] then - echo "Please update galaxy.yml version to match the KUBESPRAY_VERSION. Be sure to remove the \"v\" to adhere" - echo "to semenatic versioning" - exit 1 -fi - -if [[ $KUBESPRAY_VERSION != "${version_from_docs}" ]] -then - echo "Please update the documentation for Ansible collections under docs/ansible_collection.md to reflect the KUBESPRAY_VERSION" + echo "Please update galaxy.yml version to match the next KUBESPRAY_VERSION." + echo "Be sure to remove the \"v\" to adhere to semantic versioning" exit 1 fi From 747d8bb4c2d31669b2d7eed2b38bc4da2c689fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Trellu?= Date: Thu, 11 Jan 2024 22:03:35 -0500 Subject: [PATCH 018/103] Fix ntp installation on SLES and openSUSE (#10786) --- roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml b/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml index ad00df3c252..da4b312ebf6 100644 --- a/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml +++ b/roles/kubernetes/preinstall/tasks/0081-ntp-configurations.yml @@ -30,7 +30,7 @@ ntp_service_name: >- {% if ntp_package == "chrony" -%} chronyd - {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat"] -%} + {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse"] -%} ntpd {%- else -%} ntp From 50fbfa2a9a8d3016cfe93e2f2df576d05374560a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Trellu?= Date: Sun, 14 Jan 2024 22:21:08 -0500 Subject: [PATCH 019/103] Fix PyYAML package name on SLES and openSUSE (#10794) --- roles/kubernetes-apps/helm/vars/suse.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/helm/vars/suse.yml b/roles/kubernetes-apps/helm/vars/suse.yml index ba3964da9a4..c34bb69cee6 100644 --- a/roles/kubernetes-apps/helm/vars/suse.yml +++ b/roles/kubernetes-apps/helm/vars/suse.yml @@ -1,2 +1,2 @@ --- -pyyaml_package: python3-pyyaml +pyyaml_package: python3-PyYAML From eb405233882cfcebbfce708b1ca2743e49d7b842 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 15 Jan 2024 09:29:37 +0100 Subject: [PATCH 020/103] Convert issue templates to issues forms (#10793) * Convert the bug-report template to issue form * Convert the enchancement issue template to form * Convert "Failing Test" template to issue form * github: Remove support request template, direct to slack instead --- .github/ISSUE_TEMPLATE/bug-report.md | 44 --------- .github/ISSUE_TEMPLATE/bug-report.yaml | 117 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/enhancement.md | 11 --- .github/ISSUE_TEMPLATE/enhancement.yaml | 20 ++++ .github/ISSUE_TEMPLATE/failing-test.md | 20 ---- .github/ISSUE_TEMPLATE/failing-test.yaml | 41 ++++++++ .github/ISSUE_TEMPLATE/support.md | 18 ---- 8 files changed, 183 insertions(+), 93 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yaml create mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/enhancement.md create mode 100644 .github/ISSUE_TEMPLATE/enhancement.yaml delete mode 100644 .github/ISSUE_TEMPLATE/failing-test.md create mode 100644 .github/ISSUE_TEMPLATE/failing-test.yaml delete mode 100644 .github/ISSUE_TEMPLATE/support.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md deleted file mode 100644 index 3ead734e727..00000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -name: Bug Report -about: Report a bug encountered while operating Kubernetes -labels: kind/bug - ---- - - -**Environment**: -- **Cloud provider or hardware configuration:** - -- **OS (`printf "$(uname -srm)\n$(cat /etc/os-release)\n"`):** - -- **Version of Ansible** (`ansible --version`): - -- **Version of Python** (`python --version`): - - -**Kubespray version (commit) (`git rev-parse --short HEAD`):** - - -**Network plugin used**: - - -**Full inventory with variables (`ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"`):** - - -**Command used to invoke ansible**: - - -**Output of ansible run**: - - -**Anything else do we need to know**: - diff --git a/.github/ISSUE_TEMPLATE/bug-report.yaml b/.github/ISSUE_TEMPLATE/bug-report.yaml new file mode 100644 index 00000000000..20aed31a89b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yaml @@ -0,0 +1,117 @@ +--- +name: Bug Report +description: Report a bug encountered while operating Kubernetes +labels: kind/bug +body: + - type: markdown + attributes: + value: | + Please, be ready for followup questions, and please respond in a timely + manner. If we can't reproduce a bug or think a feature already exists, we + might close your issue. If we're wrong, PLEASE feel free to reopen it and + explain why. + - type: textarea + id: problem + attributes: + label: What happened? + description: | + Please provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. + validations: + required: true + - type: textarea + id: expected + attributes: + label: What did you expect to happen? + validations: + required: true + + - type: textarea + id: repro + attributes: + label: How can we reproduce it (as minimally and precisely as possible)? + validations: + required: true + + - type: markdown + attributes: + value: '### Environment' + + - type: textarea + id: os + attributes: + label: OS + placeholder: 'printf "$(uname -srm)\n$(cat /etc/os-release)\n"' + validations: + required: true + + - type: textarea + id: ansible_version + attributes: + label: Version of Ansible + placeholder: 'ansible --version' + validations: + required: true + + - type: input + id: python_version + attributes: + label: Version of Python + placeholder: 'python --version' + validations: + required: true + + - type: input + id: kubespray_version + attributes: + label: Version of Kubespray (commit) + placeholder: 'git rev-parse --short HEAD' + validations: + required: true + + - type: dropdown + id: network_plugin + attributes: + label: Network plugin used + options: + - calico + - cilium + - cni + - custom_cni + - flannel + - kube-ovn + - kube-router + - macvlan + - meta + - multus + - ovn4nfv + - weave + validations: + required: true + + - type: textarea + id: inventory + attributes: + label: Full inventory with variables + placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"' + + - type: input + id: ansible_command + attributes: + label: Command used to invoke ansible + + - type: textarea + id: ansible_output + attributes: + label: Output of ansible run + description: We recommend using snippets services like https://gist.github.com/ etc. + + - type: textarea + id: anything_else + attributes: + label: Anything else we need to know + description: | + By running scripts/collect-info.yaml you can get a lot of useful informations. + Script can be started by: + ansible-playbook -i -u -e ansible_ssh_user= -b --become-user=root -e dir=`pwd` scripts/collect-info.yaml + (If you using CoreOS remember to add '-e ansible_python_interpreter=/opt/bin/python'). + After running this command you can find logs in `pwd`/logs.tar.gz. You can even upload somewhere entire file and paste link here diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..aa9c1813005 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +--- +contact_links: + - name: Support Request + url: https://kubernetes.slack.com/channels/kubespray + about: Support request or question relating to Kubernetes diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md deleted file mode 100644 index 3680571358f..00000000000 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: Enhancement Request -about: Suggest an enhancement to the Kubespray project -labels: kind/feature - ---- - - -**What would you like to be added**: - -**Why is this needed**: diff --git a/.github/ISSUE_TEMPLATE/enhancement.yaml b/.github/ISSUE_TEMPLATE/enhancement.yaml new file mode 100644 index 00000000000..c0232069e9b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement.yaml @@ -0,0 +1,20 @@ +--- +name: Enhancement Request +description: Suggest an enhancement to the Kubespray project +labels: kind/feature +body: + - type: markdown + attributes: + value: Please only use this template for submitting enhancement requests + - type: textarea + id: what + attributes: + label: What would you like to be added + validations: + required: true + - type: textarea + id: why + attributes: + label: Why is this needed + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/failing-test.md b/.github/ISSUE_TEMPLATE/failing-test.md deleted file mode 100644 index cb4f1a74fff..00000000000 --- a/.github/ISSUE_TEMPLATE/failing-test.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Failing Test -about: Report test failures in Kubespray CI jobs -labels: kind/failing-test - ---- - - - -**Which jobs are failing**: - -**Which test(s) are failing**: - -**Since when has it been failing**: - -**Testgrid link**: - -**Reason for failure**: - -**Anything else we need to know**: diff --git a/.github/ISSUE_TEMPLATE/failing-test.yaml b/.github/ISSUE_TEMPLATE/failing-test.yaml new file mode 100644 index 00000000000..9a19d8485c8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/failing-test.yaml @@ -0,0 +1,41 @@ +--- +name: Failing Test +description: Report test failures in Kubespray CI jobs +labels: kind/failing-test +body: + - type: markdown + attributes: + value: Please only use this template for submitting reports about failing tests in Kubespray CI jobs + - type: textarea + id: failing_jobs + attributes: + label: Which jobs are failing ? + validations: + required: true + + - type: textarea + id: failing_tests + attributes: + label: Which tests are failing ? + validations: + required: true + + - type: input + id: since_when + attributes: + label: Since when has it been failing ? + validations: + required: true + + - type: textarea + id: failing_tests + attributes: + label: Reason for failure + description: If you don't know and have no guess, just put "Unknown" + validations: + required: true + + - type: textarea + id: anything_else + attributes: + label: Anything else we need to know diff --git a/.github/ISSUE_TEMPLATE/support.md b/.github/ISSUE_TEMPLATE/support.md deleted file mode 100644 index ddec84127bf..00000000000 --- a/.github/ISSUE_TEMPLATE/support.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: Support Request -about: Support request or question relating to Kubespray -labels: kind/support - ---- - - From 06a2a3ed6cf6ee072289b384081626b1bebc7e40 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 15 Jan 2024 12:28:56 +0100 Subject: [PATCH 021/103] Fix issues forms typos (#10796) --- .github/ISSUE_TEMPLATE/bug-report.yaml | 2 +- .github/ISSUE_TEMPLATE/failing-test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yaml b/.github/ISSUE_TEMPLATE/bug-report.yaml index 20aed31a89b..0a2ceda9a57 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yaml +++ b/.github/ISSUE_TEMPLATE/bug-report.yaml @@ -1,6 +1,6 @@ --- name: Bug Report -description: Report a bug encountered while operating Kubernetes +description: Report a bug encountered while using Kubespray labels: kind/bug body: - type: markdown diff --git a/.github/ISSUE_TEMPLATE/failing-test.yaml b/.github/ISSUE_TEMPLATE/failing-test.yaml index 9a19d8485c8..94eb1bb784e 100644 --- a/.github/ISSUE_TEMPLATE/failing-test.yaml +++ b/.github/ISSUE_TEMPLATE/failing-test.yaml @@ -28,7 +28,7 @@ body: required: true - type: textarea - id: failing_tests + id: failure_reason attributes: label: Reason for failure description: If you don't know and have no guess, just put "Unknown" From 3f78bf929808d67f426ce2248b0a555cd4397591 Mon Sep 17 00:00:00 2001 From: Louis Tu <92532497+tu1h@users.noreply.github.com> Date: Tue, 16 Jan 2024 12:23:00 +0800 Subject: [PATCH 022/103] Fix incorrect ciliumcli binary (#10575) Signed-off-by: tu1h --- roles/kubespray-defaults/defaults/main/download.yml | 2 +- roles/network_plugin/cilium/tasks/install.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 5feb32d9e8d..70744cebe32 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -695,7 +695,7 @@ downloads: enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool }}" file: true version: "{{ cilium_cli_version }}" - dest: "{{ local_release_dir }}/cilium-{{ cilium_cli_version }}-{{ image_arch }}" + dest: "{{ local_release_dir }}/cilium-{{ cilium_cli_version }}-{{ image_arch }}.tar.gz" sha256: "{{ ciliumcli_binary_checksum }}" url: "{{ ciliumcli_download_url }}" unarchive: true diff --git a/roles/network_plugin/cilium/tasks/install.yml b/roles/network_plugin/cilium/tasks/install.yml index 7678e7d4aa4..d531d72417a 100644 --- a/roles/network_plugin/cilium/tasks/install.yml +++ b/roles/network_plugin/cilium/tasks/install.yml @@ -91,7 +91,7 @@ - name: Cilium | Copy Ciliumcli binary from download dir copy: - src: "{{ downloads.ciliumcli.dest }}" + src: "{{ local_release_dir }}/cilium" dest: "{{ bin_dir }}/cilium" mode: 0755 remote_src: yes From beb2660aa8d9a8df075d44500cdc55ed2df1504b Mon Sep 17 00:00:00 2001 From: qlijin Date: Tue, 16 Jan 2024 12:23:09 +0800 Subject: [PATCH 023/103] Update docs for crio (#10785) --- docs/cri-o.md | 16 ++++++++++++++++ inventory/sample/group_vars/all/cri-o.yml | 3 +++ 2 files changed, 19 insertions(+) diff --git a/docs/cri-o.md b/docs/cri-o.md index 2405697d30d..adb837ee774 100644 --- a/docs/cri-o.md +++ b/docs/cri-o.md @@ -42,6 +42,22 @@ crio_registries: [CRI-O]: https://cri-o.io/ +The following is a method to enable insecure registries. + +```yaml +crio_insecure_registries: + - 10.0.0.2:5000 +``` + +And you can config authentication for these registries after `crio_insecure_registries`. + +```yaml +crio_registry_auth: + - registry: 10.0.0.2:5000 + username: user + password: pass +``` + ## Note about user namespaces CRI-O has support for user namespaces. This feature is optional and can be enabled by setting the following two variables. diff --git a/inventory/sample/group_vars/all/cri-o.yml b/inventory/sample/group_vars/all/cri-o.yml index 3e6e4eebb31..757dab84c93 100644 --- a/inventory/sample/group_vars/all/cri-o.yml +++ b/inventory/sample/group_vars/all/cri-o.yml @@ -1,5 +1,8 @@ +# Registries defined within cri-o. # crio_insecure_registries: # - 10.0.0.2:5000 + +# Auth config for the registries # crio_registry_auth: # - registry: 10.0.0.2:5000 # username: user From 7b7c9f509e34f26e9a7ea998916742cb1780219e Mon Sep 17 00:00:00 2001 From: "lobiyed.karim" Date: Tue, 16 Jan 2024 10:04:47 +0100 Subject: [PATCH 024/103] Add PodDisruptionBudget for CoreDNS deployment. Allows users to control disruption behavior and set maximum unavailable pods (#10557) --- roles/kubernetes-apps/ansible/defaults/main.yml | 5 +++++ roles/kubernetes-apps/ansible/tasks/coredns.yml | 4 ++++ .../ansible/templates/coredns-poddisruptionbudget.yml.j2 | 9 +++++++++ 3 files changed, 18 insertions(+) create mode 100644 roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 52444b08703..0050ce05b75 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -18,6 +18,11 @@ coredns_default_zone_cache_block: | cache 30 coredns_host_network: false coredns_port: 53 + +coredns_pod_disruption_budget: false +# value for coredns pdb +coredns_pod_disruption_budget_max_unavailable: "30%" + # coredns_additional_configs adds any extra configuration to coredns # coredns_additional_configs: | # whoami diff --git a/roles/kubernetes-apps/ansible/tasks/coredns.yml b/roles/kubernetes-apps/ansible/tasks/coredns.yml index a5f7b198cae..8d5aad25531 100644 --- a/roles/kubernetes-apps/ansible/tasks/coredns.yml +++ b/roles/kubernetes-apps/ansible/tasks/coredns.yml @@ -14,6 +14,7 @@ - { name: dns-autoscaler, file: dns-autoscaler.yml, type: deployment } - { name: dns-autoscaler, file: dns-autoscaler-clusterrole.yml, type: clusterrole } - { name: dns-autoscaler, file: dns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding } + - { name: coredns, file: coredns-poddisruptionbudget.yml, type: poddisruptionbudget, condition: coredns_pod_disruption_budget } - { name: dns-autoscaler, file: dns-autoscaler-sa.yml, type: sa } register: coredns_manifests vars: @@ -22,6 +23,7 @@ - dns_mode in ['coredns', 'coredns_dual'] - inventory_hostname == groups['kube_control_plane'][0] - enable_dns_autoscaler or item.name != 'dns-autoscaler' + - item.condition | default(True) tags: - coredns @@ -34,6 +36,7 @@ - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment } - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc } - { name: dns-autoscaler, src: dns-autoscaler.yml, file: coredns-autoscaler-secondary.yml, type: deployment } + - { name: coredns, file: coredns-poddisruptionbudget.yml, type: poddisruptionbudget, condition: coredns_pod_disruption_budget } register: coredns_secondary_manifests vars: clusterIP: "{{ skydns_server_secondary }}" @@ -42,5 +45,6 @@ - dns_mode == 'coredns_dual' - inventory_hostname == groups['kube_control_plane'][0] - enable_dns_autoscaler or item.name != 'dns-autoscaler' + - item.condition | default(True) tags: - coredns diff --git a/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 new file mode 100644 index 00000000000..7df6b262186 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/coredns-poddisruptionbudget.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: coredns{{ coredns_ordinal_suffix }} +spec: + maxUnavailable: {{ coredns_pod_disruption_budget_max_unavailable }} + selector: + matchLabels: + k8s-app: kube-dns{{ coredns_ordinal_suffix }} From a0a2f402955ecfc72ecfa8c0b2948d00da464dea Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Tue, 16 Jan 2024 21:15:53 +0800 Subject: [PATCH 025/103] add containerd config override_path (#10776) --- roles/container-engine/containerd/templates/hosts.toml.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/container-engine/containerd/templates/hosts.toml.j2 b/roles/container-engine/containerd/templates/hosts.toml.j2 index c04dc478995..ea003ed44e7 100644 --- a/roles/container-engine/containerd/templates/hosts.toml.j2 +++ b/roles/container-engine/containerd/templates/hosts.toml.j2 @@ -2,7 +2,6 @@ server = "https://{{ item.prefix }}" {% for mirror in item.mirrors %} [host."{{ mirror.host }}"] capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"] -{% if mirror.skip_verify is defined %} skip_verify = {{ mirror.skip_verify | default('false') | string | lower }} -{% endif %} + override_path = {{ mirror.override_path | default('false') | string | lower }} {% endfor %} From a9e29a9eb2e0e01e50760281a05a2247def44b4d Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 16 Jan 2024 15:50:41 +0100 Subject: [PATCH 026/103] Fix etcd client generation (#10769) * ci: redefine multinode to node-etcd-client This should allow to catch several class of problem rather than just one -> from network plugin such as calico or cilium talking directly to the etcd. * Dynamically define etcd host range This has two benefits: - We don't play the etcd role twice for no reason - We have access to the whole cluster (if needed) to use things like group_by. --- docs/test_cases.md | 5 +-- playbooks/install_etcd.yml | 34 +++++++++---------- .../roles/packet-ci/templates/inventory.j2 | 6 +++- .../roles/packet-ci/vars/main.yml | 2 +- .../packet_ubuntu22-calico-etcd-datastore.yml | 2 +- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/docs/test_cases.md b/docs/test_cases.md index d5aef62f2a7..b26d3aa50f4 100644 --- a/docs/test_cases.md +++ b/docs/test_cases.md @@ -1,6 +1,6 @@ # Node Layouts -There are six node layout types: `default`, `separate`, `ha`, `scale`, `all-in-one`, and `multinode`. +There are six node layout types: `default`, `separate`, `ha`, `scale`, `all-in-one`, and `node-etcd-client`. `default` is a non-HA two nodes setup with one separate `kube_node` and the `etcd` group merged with the `kube_control_plane`. @@ -18,7 +18,8 @@ never actually deployed, but certificates are generated for them. `all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged. -`multinode` layout consists of two separate `kube_node` and a merged single `etcd+kube_control_plane` node. +`node-etcd-client` layout consists of a 4 nodes cluster, all of them in `kube_node`, first 3 in `etcd` and only one `kube_control_plane`. +This is necessary to tests setups requiring that nodes are etcd clients (use of cilium as `network_plugin` for instance) Note, the canal network plugin deploys flannel as well plus calico policy controller. diff --git a/playbooks/install_etcd.yml b/playbooks/install_etcd.yml index 9bd13dfcd6e..b8e4d1d89b0 100644 --- a/playbooks/install_etcd.yml +++ b/playbooks/install_etcd.yml @@ -1,20 +1,21 @@ --- -- name: Install etcd - hosts: etcd:kube_control_plane - gather_facts: False - any_errors_fatal: "{{ any_errors_fatal | default(true) }}" - environment: "{{ proxy_disable_env }}" +- name: Add worker nodes to the etcd play if needed + hosts: kube_node roles: - { role: kubespray-defaults } - - role: etcd + tasks: + - name: Check if nodes needs etcd client certs (depends on network_plugin) + group_by: + key: "_kubespray_needs_etcd" + when: + - kube_network_plugin in ["flannel", "canal", "cilium"] or + (cilium_deploy_additionally | default(false)) or + (kube_network_plugin == "calico" and calico_datastore == "etcd") + - etcd_deployment_type != "kubeadm" tags: etcd - vars: - etcd_cluster_setup: true - etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" - when: etcd_deployment_type != "kubeadm" -- name: Install etcd certs on nodes if required - hosts: k8s_cluster +- name: Install etcd + hosts: etcd:kube_control_plane:_kubespray_needs_etcd gather_facts: False any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" @@ -23,9 +24,6 @@ - role: etcd tags: etcd vars: - etcd_cluster_setup: false - etcd_events_cluster_setup: false - when: - - etcd_deployment_type != "kubeadm" - - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool - - kube_network_plugin != "calico" or calico_datastore == "etcd" + etcd_cluster_setup: true + etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" + when: etcd_deployment_type != "kubeadm" diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 index da7e749691c..cb697762944 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 @@ -81,16 +81,20 @@ instance-2 [broken_etcd] instance-1 etcd_member_name=etcd2 instance-2 etcd_member_name=etcd3 -{% elif mode == "multinode" %} +{% elif mode == "node-etcd-client" %} [kube_control_plane] instance-1 [etcd] instance-1 +instance-2 +instance-3 [kube_node] +instance-1 instance-2 instance-3 +instance-4 {% endif %} [k8s_cluster:children] diff --git a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml index 5cfe561b707..81aa17d9fc4 100644 --- a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml @@ -7,7 +7,7 @@ _vm_count_dict: ha-recover: 3 ha-recover-noquorum: 3 all-in-one: 1 - multinode: 3 + node-etcd-client: 4 default: 2 vm_count: "{{ _vm_count_dict[mode | d('default')] }}" diff --git a/tests/files/packet_ubuntu22-calico-etcd-datastore.yml b/tests/files/packet_ubuntu22-calico-etcd-datastore.yml index c95a7602fd9..e2d3cb72ae2 100644 --- a/tests/files/packet_ubuntu22-calico-etcd-datastore.yml +++ b/tests/files/packet_ubuntu22-calico-etcd-datastore.yml @@ -1,7 +1,7 @@ --- # Instance settings cloud_image: ubuntu-2204 -mode: multinode +mode: node-etcd-client vm_memory: 1600Mi # Kubespray settings From b7a83531e7ad1553a641033d0bd7f085b311843b Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 17 Jan 2024 09:50:48 +0100 Subject: [PATCH 027/103] etcd: update to v3.5.10 (#10798) --- README.md | 2 +- roles/kubespray-defaults/defaults/main/download.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8362ad5bca3..4a594a61954 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ Note: Upstart/SysV init based OS types are not supported. - Core - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.5 - - [etcd](https://github.com/etcd-io/etcd) v3.5.9 + - [etcd](https://github.com/etcd-io/etcd) v3.5.10 - [docker](https://www.docker.com/) v20.10 (see note) - [containerd](https://containerd.io/) v1.7.11 - [cri-o](http://cri-o.io/) v1.27 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 70744cebe32..4313391cc22 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -134,9 +134,9 @@ skopeo_version: "v1.13.2" kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" etcd_supported_versions: - v1.28: "v3.5.9" - v1.27: "v3.5.9" - v1.26: "v3.5.9" + v1.28: "v3.5.10" + v1.27: "v3.5.10" + v1.26: "v3.5.10" etcd_version: "{{ etcd_supported_versions[kube_major_version] }}" crictl_supported_versions: From 669589f7614a311c934e130fe1d5bdf6ed485a7b Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Wed, 17 Jan 2024 17:03:02 +0800 Subject: [PATCH 028/103] cleanup-for-2.23.2 (#10800) --- .gitlab-ci.yml | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 180166c2422..e5d17bc923f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ stages: - deploy-special variables: - KUBESPRAY_VERSION: v2.23.1 + KUBESPRAY_VERSION: v2.23.2 FAILFASTCI_NAMESPACE: 'kargo-ci' GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray' ANSIBLE_FORCE_COLOR: "true" diff --git a/README.md b/README.md index 4a594a61954..98c2f697e88 100644 --- a/README.md +++ b/README.md @@ -75,11 +75,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou to access the inventory and SSH key in the container, like this: ```ShellSession -git checkout v2.23.1 -docker pull quay.io/kubespray/kubespray:v2.23.1 +git checkout v2.23.2 +docker pull quay.io/kubespray/kubespray:v2.23.2 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ - quay.io/kubespray/kubespray:v2.23.1 bash + quay.io/kubespray/kubespray:v2.23.2 bash # Inside the container you may now run the kubespray playbooks: ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml ``` From 78eb74c252992acb99dbf7d2a62b04ebce662043 Mon Sep 17 00:00:00 2001 From: ERIK Date: Fri, 19 Jan 2024 12:55:41 +0800 Subject: [PATCH 029/103] Pin equinix provider version (#10809) Signed-off-by: bo.jiang --- contrib/terraform/equinix/provider.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/terraform/equinix/provider.tf b/contrib/terraform/equinix/provider.tf index 61c0aba12d5..1c6b8b45cad 100644 --- a/contrib/terraform/equinix/provider.tf +++ b/contrib/terraform/equinix/provider.tf @@ -7,7 +7,7 @@ terraform { required_providers { equinix = { source = "equinix/equinix" - version = "~> 1.14" + version = "1.24.0" } } } From 64447e745e53d3f486356d03ecb195729a302ea1 Mon Sep 17 00:00:00 2001 From: Mohamed Omar Zaian Date: Fri, 19 Jan 2024 09:07:27 +0100 Subject: [PATCH 030/103] [kubernetes] Make kubernetes v1.28.6 default (#10810) --- README.md | 2 +- .../group_vars/k8s_cluster/k8s-cluster.yml | 2 +- .../defaults/main/checksums.yml | 47 +++++++++++++++++++ .../kubespray-defaults/defaults/main/main.yml | 2 +- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 98c2f697e88..c3f149db5e4 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Note: Upstart/SysV init based OS types are not supported. ## Supported Components - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.5 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.6 - [etcd](https://github.com/etcd-io/etcd) v3.5.10 - [docker](https://www.docker.com/) v20.10 (see note) - [containerd](https://containerd.io/) v1.7.11 diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 8153a04cde7..454ba303f22 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.28.5 +kube_version: v1.28.6 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index 7f046f475a4..4ad16c1acd0 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -73,7 +73,9 @@ crio_archive_checksums: # Kubernetes versions above Kubespray's current target version are untested and should be used with caution. kubelet_checksums: arm: + v1.29.1: 0 v1.29.0: 0 + v1.28.6: 0 v1.28.5: 0 v1.28.4: 0 v1.28.3: 0 @@ -90,6 +92,7 @@ kubelet_checksums: v1.27.2: 0 v1.27.1: 0 v1.27.0: 0 + v1.26.13: 74a0e15e47f82e6b073e6fa3eac3ebd2d248bf29a902ee3f0158f204509ca5d6 v1.26.12: 9bfa6d78d4f6a865d7a1731624d5aafdc69dff54139eed6c2f076158a67c904a v1.26.11: a7bbe545ab23b068d5e25262bf17a10039edb5c073ba896d036af12afb5fbf9c v1.26.10: 3130ae14dda009f92296045d9cc2a1f566fa7d85905d0f6a8db47b05496c5d53 @@ -104,13 +107,16 @@ kubelet_checksums: v1.26.1: fe940be695f73c03275f049cb17f2bf2eb137014334930ce5c6de12573c1f21f v1.26.0: cabf702fc542fcbb1173c713f1cbec72fd1d9ded36cdcdbbd05d9c308d8360d1 arm64: + v1.29.1: e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d v1.29.0: 0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16 + v1.28.6: ee2c060deff330d3338e24aec9734c9e5d5aea4fea1905c0795bccff6997a65e v1.28.5: 28ddb696eb6e076f2a2f59ccaa2e409785a63346e5bda819717c6e0f58297702 v1.28.4: bf203989dd9b3987b8a0d2331dcce6319f834b57df810fafba5a4805d54823ac v1.28.3: 64f56e9c55183919153fe59df2c9015dff09c56de13a3cbccc0f04a95b76dab9 v1.28.2: 32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577 v1.28.1: 9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c v1.28.0: 05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9 + v1.27.10: 0edadc44ef36be8d8106cad9972360c0477540e2d8c0bbeb38fd97fd1d7801d5 v1.27.9: 8a14bc3739f5ca3b23d08301c2e769ee58c8d1cecb7243b46b1c098ae77effd7 v1.27.8: 71849182ceb018dc084f499ad28b7b1afb7f23e35ccaf8421941dd5dafef0d4c v1.27.7: ed5bfa48ee64d5e6cf23ed9fc03ea0593021839429fdc1ea7cc2ebf3f11b6491 @@ -121,6 +127,7 @@ kubelet_checksums: v1.27.2: 810cd9a611e9f084e57c9ee466e33c324b2228d4249ff38c2588a0cc3224f10d v1.27.1: dbb09d297d924575654db38ed2fc627e35913c2d4000c34613ac6de4995457d0 v1.27.0: 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157 + v1.26.13: 4925b769336df7cf5e7a7f33d82ee5c69ac67a7680ac1f7064534f971d9a79ab v1.26.12: d4406ed5bfd12768c03fc4fbe011a01e5c91b74d1d4b526fe3ac320d13295ffb v1.26.11: ff8940394446028e75a2b8155e22eccf635f6a128f45dee41e293493d2743d17 v1.26.10: ddebcc1af7f203a2ee3d80dad0baaf84a4680748839f5583b39cbce4b8afa7f2 @@ -135,13 +142,16 @@ kubelet_checksums: v1.26.1: f4b514162b52d19909cf0ddf0b816d8d7751c5f1de60eda90cd84dcccc56c399 v1.26.0: fb033c1d079cac8babb04a25abecbc6cc1a2afb53f56ef1d73f8dc3b15b3c09e amd64: + v1.29.1: 1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3 v1.29.0: e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05 + v1.28.6: 8506df1f20a5f8bba0592f5a4cf5d0cc541047708e664cb88580735400d0b26f v1.28.5: bf37335da58182783a8c63866ec1f895b4c436e3ed96bdd87fe3f8ae8004ba1d v1.28.4: db2a473b73c3754d4011590f2f0aa877657608499590c6b0f8b40bec96a3e9ba v1.28.3: a3a058b4ba30da01ffe1801cd38fcad58a9022a2d39e080b4b2e0e9749a75ad5 v1.28.2: 17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185 v1.28.1: 2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3 v1.28.0: bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981 + v1.27.10: 25a34bf98bb8a296ea07f1ebbcb496b1e6b6c6da3247695288a7c99fc8c1be2c v1.27.9: ede60eea3acbac3f35dbb23d7b148f45cf169ebbb20af102d3ce141fc0bac60c v1.27.8: 2e0557b38c5b9a1263eed25a0b84d741453ed9c0c7bd916f80eadaf7edfb7784 v1.27.7: 236bc8bc22c52e914d3364c23e273628c63e193365b6a43b8cb013716c1cd2f5 @@ -152,6 +162,7 @@ kubelet_checksums: v1.27.2: a0d12afcab3b2836de4a427558d067bebdff040e9b306b0512c93d9d2a066579 v1.27.1: cb2845fff0ce41c400489393da73925d28fbee54cfeb7834cd4d11e622cbd3a7 v1.27.0: 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29 + v1.26.13: 05860ef65deb594dc72034c0614f93d7ba9d1f229ff73b43c484298e22465f1b v1.26.12: aed0a351b01f1e6a84a0992ef1265bb0c9994b900162c075df58d0d02517d3df v1.26.11: a62953f20fa9fedff50c6c5423e68981e3382d92cf04174d5bca5f4d084de0c5 v1.26.10: 4c27b3a9f332a6762f7240d0784c64775d4db5a1b881eeae05c4561d06c267ec @@ -166,13 +177,16 @@ kubelet_checksums: v1.26.1: 8b99dd73f309ca1ac4005db638e82f949ffcfb877a060089ec0e729503db8198 v1.26.0: b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5 ppc64le: + v1.29.1: 467d2b457205363f53f72081295ea390fc25215b0ccc29dc04c4f82925266067 v1.29.0: 67f09f866d3e4aee8211ce9887ec8bc427b188474a882a7af999fc0fee939028 + v1.28.6: 8f79f40bef88aaedfdf7256de48a972295b0069ae0ddefa90dff3f8690c825ce v1.28.5: ae9fe81804ba67ee81e8a5fe1dc18fe285267764c61f831886a25245a11d8528 v1.28.4: d79c97811fb10c1b1f48b69573f1164f108630631d9dba0d991fe924bd305f20 v1.28.3: f20cfb8c9de73cdc66fbbecd03bb936ce57fe86ebced8ea93aa64ebda0235c21 v1.28.2: 79f568ac700d29f88d669c6b6a09adb3b726bdd13c10aa0839cbc70b414372e5 v1.28.1: 547fc76f0c1d78352fad841ebeacd387fe48750b2648565dfd49197621622fbb v1.28.0: 22de59965f2d220afa24bf04f4c6d6b65a4bb1cd80756c13381973b1ac3b4578 + v1.27.10: c5014bed224347245fadec3d763846ec33ccd7a580d0c4ee19a45a948392f20c v1.27.9: f270051c9b0f36da10a5d27011783be042edd396e8c729709c2396f29b72b6d2 v1.27.8: 2354fdb19b5018cabe43fde1979965686afd3c95b75531e678a0064c4a30b4e9 v1.27.7: 9154a7b5d5793fb8a05cba0d309ddfd975409d0e0686be19cf2feca172f06162 @@ -183,6 +197,7 @@ kubelet_checksums: v1.27.2: 3af92edd687f7932e7fce877944dfe5efa437bf5f171fc8331725c631a1a86ef v1.27.1: 7a800b9539beaba0b5d6357070a40fb3c4d216c2ad6693b15f9b1307b1c99e1f v1.27.0: 17c061a9f7919697ac71c151c19337f65b86f59f59441687ac92e977d851c75b + v1.26.13: 5ebb3cf365b4b5a70de859cb8a5b5c6abb359fa4ffe101a5f960c99f2342e79e v1.26.12: c740ea116206a3e9420b1ab810991c21910abeea947da1b295c2454700cdbe44 v1.26.11: 408e47e33159551ebeeb6f4ef836bd7d07fc92d16ca6d662c85ea8fa0fe34a8e v1.26.10: 5cc1046c8f9188dc6d5ec22200eeed7c8c1ae88a23ce5d79e8dfaf1ff61f2bf8 @@ -198,13 +213,16 @@ kubelet_checksums: v1.26.0: df13099611f4eada791e5f41948ef94f7b52c468dff1a6fcad54f4c56b467a7f kubectl_checksums: arm: + v1.29.1: a4b478cc0e9adaab0c5bb3627c20c5228ea0fe2aeff9e805d611eb3edb761972 v1.29.0: a2388eb458d07ec734e4fa02fd0147456a1922a7d6b8e67a32db9d64a4d7621c + v1.28.6: 2358d98d4970c177a3af0ae1c2398f69922074a961a61cdff6ae4a7f13106dc1 v1.28.5: 0819c9d0ea66a1e20d74d9a455090e1f67fe07d671866be342ab55532203f4bc v1.28.4: 835ef8d72f8dec4493b855ddd8e4163f107053496d923c89c216489a45757df6 v1.28.3: b252ec9e97abde80fe067eb215a1acb69a8c83022cba897fd2c4d387bd45f5ca v1.28.2: 6576aa70413ff00c593a07b549b8b9d9e5ef73c42bb39ab4af475e0fdb540613 v1.28.1: eaa05dab1bffb8593d8e5caa612530ee5c914ee2be73429b7ce36c3becad893f v1.28.0: 372c4e7bbe98c7067c4b7820c4a440c931ad77f7cb83d3237b439ca3c14d3d37 + v1.27.10: 4d81649935ec127f9aa21954697f82e0796f61e8e6406fd058b3a8b80e858c8e v1.27.9: 89b76aa415018377f2c5fc33fc4d45f4997cc63677336f1768ee8a11593515ce v1.27.8: 2f2936f950beb3f08ee0e45fbf80d020163829b95aa11c99ec726ee1a922329c v1.27.7: 2a387ade64cd4cf90f002ae8c0e6f60250d26d1ee2cce6051f559430a44352ea @@ -215,6 +233,7 @@ kubectl_checksums: v1.27.2: 7792f5630543c0af84f444521ee6113da5ae00f2b50872d57324aa725a5341c5 v1.27.1: fe704e355bf2c5f69964cd12772687535a11a5e9ec0baf4f27e0a8fb156bc615 v1.27.0: 288470e3eb89a2f55273d753ce6674dfb00e732f2971428acb964810aa726188 + v1.26.13: 1a073f6f50e4afae000e286200e8face5c5e8b8545d8f70b163d4a2431ab096a v1.26.12: cf788244170a71dc6540abccdff76327a48d8c5fbc88ffde9e1ebaa4616f8d2e v1.26.11: 34cc371daef6e42aea27117aa643619ab870d1cb61995530fc9bcb992e3caf10 v1.26.10: d26ab68b4884eec4dcadd0f564ba197d48f21aebed509b9b1541d31f2aaf5890 @@ -229,13 +248,16 @@ kubectl_checksums: v1.26.1: e067d59ac19e287026b5c2b75a1077b1312ba82ad64ee01dff2cdafd57720f39 v1.26.0: 8eef310d0de238c582556d81ab8cbe8d6fca3c0e43ee337a905dcdd3578f9dda arm64: + v1.29.1: 96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b v1.29.0: 8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7 + v1.28.6: 0de705659a80c3fef01df43cc0926610fe31482f728b0f992818abd9bdcd2cb9 v1.28.5: f87fe017ae3ccfd93df03bf17edd4089672528107f230563b8c9966909661ef2 v1.28.4: edf1e17b41891ec15d59dd3cc62bcd2cdce4b0fd9c2ee058b0967b17534457d7 v1.28.3: 06511f03e34d8ee350bd55717845e27ebec3116526db7c60092eeb33a475a337 v1.28.2: ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53 v1.28.1: 46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6 v1.28.0: f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c + v1.27.10: 2e1996379d5a8b132e0606fcd3df3c8689e11882630b75cca3b7135126847871 v1.27.9: bda475539fdeda9d8a85a84b967af361af264d0826c121b23b0b62ee9b00cd2d v1.27.8: 97ed6739e2803e63fd2d9de78be22d5ba6205bb63179a16ec773063526525a8e v1.27.7: 61fc334f2c0290270e43fb8a1d4ff07e8cec5642d5a123eb7ab66a134b04ae83 @@ -246,6 +268,7 @@ kubectl_checksums: v1.27.2: 1b0966692e398efe71fe59f913eaec44ffd4468cc1acd00bf91c29fa8ff8f578 v1.27.1: fd3cb8f16e6ed8aee9955b76e3027ac423b6d1cc7356867310d128082e2db916 v1.27.0: f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc + v1.26.13: 4abebc34c114111b81bbf5222f0810e6899937d04bc453d9ccd77046643bbcda v1.26.12: b9cfdb6c9a53146330d5694e711c48febb7ec022aea8d16a51b5e12d761580cc v1.26.11: 4a6d2b7204af3cf84cd0e2c670fbb211501050c9a288de49de3c6363d4e0a63e v1.26.10: 5752e3908fa1d338eb1fa99a6f39c6a4c27b065cb459da84e35c4ec718879f14 @@ -260,13 +283,16 @@ kubectl_checksums: v1.26.1: 4027cb0a2840bc14ec3f18151b3360dd2d1f6ce730ed5ac28bd846c17e7d73f5 v1.26.0: 79b14e4ddada9e81d2989f36a89faa9e56f8abe6e0246e7bdc305c93c3731ea4 amd64: + v1.29.1: 69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f v1.29.0: 0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5 + v1.28.6: c8351fe0611119fd36634dd3f53eb94ec1a2d43ef9e78b92b4846df5cc7aa7e3 v1.28.5: 2a44c0841b794d85b7819b505da2ff3acd5950bd1bcd956863714acc80653574 v1.28.4: 893c92053adea6edbbd4e959c871f5c21edce416988f968bec565d115383f7b8 v1.28.3: 0c680c90892c43e5ce708e918821f92445d1d244f9b3d7513023bcae9a6246d1 v1.28.2: c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 v1.28.1: e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1 v1.28.0: 4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce + v1.27.10: bfb219643c28d9842fceae51590776f06987835d93fc3cb9b0149c9111c741ac v1.27.9: d0caae91072297b2915dd65f6ef3055d27646dce821ec67d18da35ba9a8dc85b v1.27.8: 027b3161e99fa0a7fa529e8f17f73ee2c0807c81c721ca7cf307f6b41c17bc57 v1.27.7: e5fe510ba6f421958358d3d43b3f0b04c2957d4bc3bb24cf541719af61a06d79 @@ -277,6 +303,7 @@ kubectl_checksums: v1.27.2: 4f38ee903f35b300d3b005a9c6bfb9a46a57f92e89ae602ef9c129b91dc6c5a5 v1.27.1: 7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d v1.27.0: 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87 + v1.26.13: e4bad4273431f9f5f05f27f5c2054cbbad6d9ee00b85e0810cb4ef0489b02571 v1.26.12: 8e6af8d68e7b9d2a1eb43255c0da793276e549a34a2b9c3c87a9c26438e7fd71 v1.26.11: 27c34a0870230d9dd723e1e01114634e396cd2a3d25ced263b769a4bd53e4edd v1.26.10: 93ad44b4072669237247bfbc171be816f08e7e9e4260418d2cfdd0da1704ae86 @@ -291,13 +318,16 @@ kubectl_checksums: v1.26.1: d57be22cfa25f7427cfb538cfc8853d763878f8b36c76ce93830f6f2d67c6e5d v1.26.0: b6769d8ac6a0ed0f13b307d289dc092ad86180b08f5b5044af152808c04950ae ppc64le: + v1.29.1: b7780124ccfe9640f3a37d242d31e8dbb252bcd379bd0d7bf3776d15baf15ca3 v1.29.0: ea926d8cf25e2ce982ff5c375da32b51ccbd122b721b1bc4a32f52a9a0d073ab + v1.28.6: 60fdb4386b5499dd6a6e3a369f35eef63c99647f7a0436fdbeb4db8c052d14f6 v1.28.5: 4448a9f95421cbe69726aa4d2967d706bc43466b9c656c7425b55431b1c20dd4 v1.28.4: 816ca2cef39c0d1ac8ad60c05ae6f6ea5c4a0ca33748240bd1f019381244ca23 v1.28.3: 2b7331a91f558a748167672c18458aa205d4d6d2794654dfd308942e9a376ca4 v1.28.2: 87cca30846fec99a4fbea122b21e938717b309631bd2220de52049fce30d2e81 v1.28.1: 81b45c27abbdf2be6c5203dfccfd76ded1ac273f9f7672e6dcdf3440aa191324 v1.28.0: 7a9dcb4c75b33b9dac497c1a756b1f12c7c63f86fc0f321452360fbe1a79ce0f + v1.27.10: 445928336932248cb104d99919e659696afa60f8dd8513821f92775e893d0dcb v1.27.9: 2464d947370b8902e1245b0a75a4ecf55fe2aeee5bc87f2add7da00b73535a59 v1.27.8: e25a09dea99192ff43ee13af61bfadd7c79eb538dc8e85376b6c590b4d471204 v1.27.7: a60da56f856bc5b224f0be71b03443d5a0fb69424f31cd86f069cc79c13b2870 @@ -308,6 +338,7 @@ kubectl_checksums: v1.27.2: efee037a276f72c77cc230194d7dadf943a5778be46b7985edeb414d27894266 v1.27.1: 440bcfd9611319f3d9e5d4fa4cdee2421cdf80c01fad223934d9a9b640673d75 v1.27.0: daa9f1d4fe3f217de2546bca4ac14601f34b34a25c1f571f1e44eb313aee1385 + v1.26.13: d1d9058ca73bf669d4706371a5af9340384756392608a2f264b37977d62b1efb v1.26.12: f977850aff89f7e148fbe940aa84faf7e1abbf1936527c79fc111a6f759b73b0 v1.26.11: f2c577e75277415203e699ca7bc08eb640a0b8211a0366926122e3ac4f98997c v1.26.10: 24674d7ce1ccef207d127bd952f6f40f951d7682cfa603f6989ed80e91dcaafc @@ -323,13 +354,16 @@ kubectl_checksums: v1.26.0: 9e2b2a03ee5fc726ebd39b6a09498b6508ade8831262859c280d39df33f8830d kubeadm_checksums: arm: + v1.29.1: 0 v1.29.0: 0 + v1.28.6: 0 v1.28.5: 0 v1.28.4: 0 v1.28.3: 0 v1.28.2: 0 v1.28.1: 0 v1.28.0: 0 + v1.27.10: 0 v1.27.9: 0 v1.27.8: 0 v1.27.7: 0 @@ -340,6 +374,7 @@ kubeadm_checksums: v1.27.2: 0 v1.27.1: 0 v1.27.0: 0 + v1.26.13: 2b61d87e7babc764ac53783af300dffef556f1ea9039b9ac550f23c6048ee1e1 v1.26.12: a15b212a2ec393b229d35eb0900b287c4a4c52e1993ef69eadcce1105d5c2bc9 v1.26.11: 3683f52861f89c3cf48ea7754fd2e81ad30a51bef57d056b2663e5e9139870b6 v1.26.10: b59433ebaddf5c5d816033304989f9dd2e0b863650532cd195c1d41c0f973f14 @@ -354,13 +389,16 @@ kubeadm_checksums: v1.26.1: 0dbd0a197013a3fdc5cb3e012fa8b0d50f38fd3dda56254f4648e08ac867fb60 v1.26.0: 3368537a5e78fdbfa3cbcae0d19102372a0f4eb6b6a78e7b6b187d5db86d6c9e arm64: + v1.29.1: 3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf v1.29.0: bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980 + v1.28.6: 4298cad464e92eec19cdf3e6a607a82a1d626ae70fedba7956175152ab983457 v1.28.5: 22bb6b3377204e93d008f33ac4924d77adca1478f1ae3b515c03476ba54f1adc v1.28.4: a4422780020954436b8e76ab1c59b68c5581a54432dd3e566c4709bb40c8d4f9 v1.28.3: dcb37d78ccdfe9d8dd6f100e188ddc6e3f5570d0c49db68470073683b453a1e7 v1.28.2: 010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff v1.28.1: 7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8 v1.28.0: b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab + v1.27.10: ed0447155a7e967ae23480b06b31b2c0aaa871e7c59dfd82ae25b03a1eccf6e6 v1.27.9: d3d022842b0b8e4661222e8873249f5acafdbef52fd1bfb98152a582352b3c40 v1.27.8: 0d0f5b2781d663d314e785d14361aa5a09cfaf6e1694aa3cc731f4f06342ec13 v1.27.7: 46d7c43532233906919a53ee0e03ab04ab9e08514392d17a86f058e0364cda4b @@ -371,6 +409,7 @@ kubeadm_checksums: v1.27.2: 8f01f363f7c7f92de2f2276124a895503cdc5a60ff549440170880f296b087eb v1.27.1: 024a59cd6fc76784b597c0c1cf300526e856e8c9fefa5fa7948158929b739551 v1.27.0: acd805c6783b678ee0068b9dd8165bbfd879c345fd9c25d6a978dbc965f48544 + v1.26.13: 2cce047abb5df5c9e1e48405e1d4822f2ccc6609668db49e95d9ccbbeaa71a7a v1.26.12: 2dee03d460e8a1b3f30e5ef48ce94f5eeb4ff4550e65860e6e2d94b368c2cde6 v1.26.11: a13318c1493e58a9f7c4359c79443f3c86a690ec601bcc76308c809d8d61edb8 v1.26.10: 1ddcb47ee4f7171736dbacc046a7ceae55411ee09920435c3821b530f4650428 @@ -385,13 +424,16 @@ kubeadm_checksums: v1.26.1: db101c4bb8e33bd69241de227ed317feee6d44dbd674891e1b9e11c6e8b369bb v1.26.0: 652844c9518786273e094825b74a1988c871552dc6ccf71366558e67409859d1 amd64: + v1.29.1: d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd v1.29.0: 629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c + v1.28.6: bda3eda8d51e8746a42b535b7eab7df52b091a796227c3212dc30909a8f1b431 v1.28.5: 2b54078c5ea9e85b27f162f508e0bf834a2753e52a57e896812ec3dca92fe9cd v1.28.4: b4d2531b7cddf782f59555436bc098485b5fa6c05afccdeecf0d62d21d84f5bd v1.28.3: ce3848b1dfa562e0fa2f911a3d8e3bb07ba040eea76654d68e213315c8846ac0 v1.28.2: 6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00 v1.28.1: 6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736 v1.28.0: 12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb + v1.27.10: 23985e958443ac1aabdbeeedc675358abc0638eb580707829fd42b0996a0aae5 v1.27.9: 78dddac376fa2f04116022cb44ed39ccb9cb0104e05c5b21b220d5151e5c0f86 v1.27.8: f8864769b8b2d7a14f53eb983f23317ff14d68ab76aba71e9de17ce84c38d4eb v1.27.7: bc589219a003b3b94c114e4bcf20549a02657a0c6e5c73f588b37817148892d2 @@ -402,6 +444,7 @@ kubeadm_checksums: v1.27.2: 95c4bfb7929900506a42de4d92280f06efe6b47e0a32cbc1f5a1ed737592977a v1.27.1: c7d32d698e99b90f877025104cb4a9f3f8c707e99e6817940f260135b6d1ad0a v1.27.0: 78d0e04705a7bdb76a514d60f60c073b16334b15f57ee87f064354ca8a233e80 + v1.26.13: 62c0e49f749d230f0c27fa53cbe4851894d9002757fd3653fc5da36aa7c68032 v1.26.12: 5a5d65acefb50010859be8ffba8e6e059d552ae357e3101c12c62e747a9416a2 v1.26.11: 58f886e39e517ba1a92493f136e80f1b6ea9362966ad9d2accdf2133004161f2 v1.26.10: 27ed1d857f4a315f3d059168c6e25fdbf0559f9c8e59bab6c50e7921f74dadbf @@ -416,13 +459,16 @@ kubeadm_checksums: v1.26.1: 1531abfe96e2e9d8af9219192c65d04df8507a46a081ae1e101478e95d2b63da v1.26.0: 72631449f26b7203701a1b99f6914f31859583a0e247c3ac0f6aaf59ca80af19 ppc64le: + v1.29.1: 3ec6d90c05dd8e4c6bb1f42fd2fe0f091d85317efaf47d9baebd9af506b3878b v1.29.0: 4c414a463ed4277e9062c797d1c0435aa7aec2fd1688c5d34e3161c898113cb5 + v1.28.6: 71fc8af0f80599a991ece0c31b21ca85f3ce49322941a305048d9287c249446c v1.28.5: a9bf8b18711639d9d002f63cebc22c8df1627737891c640f2229461d19b8c321 v1.28.4: 24e4b42b1d0ec68fc291fcc57fa88ec34b9e8ba758e01639873ef2068222af4a v1.28.3: 0ae62912b057f3228dd7a9fbe2492c4b8c3a661f27a1d46e70b0b6627ccf60fb v1.28.2: fdc28482a4316c84d61b0997c29c4d4c7b11459af9c654fdee3b4a3031f0fcb7 v1.28.1: 73e06f2b614ed5665951f7c059e225a7b0b31319c64a3f57e146fbe7a77fe54e v1.28.0: 146fe9194486e46accd5054fa93939f9608fdbeefefc4bc68e4c40fb4a84ccc9 + v1.27.10: c928ad330bae724b1ef9775e07285408727513a024e3d86e3d72e05768859db8 v1.27.9: 92da9084fa9f8b8b55436b61ec3c697ef951b0b0416a3b3a7f0dd0e5e4d8cd88 v1.27.8: d65b972cd661cb28972f0df731f9e5b65d959920275bad5ef44ff94d3bb8331d v1.27.7: 6a95a8fb5560a6698d895b2a809485ace0cf5b8c6fb89c843ab20cf89d8b11ad @@ -433,6 +479,7 @@ kubeadm_checksums: v1.27.2: 412bccd310f4976201d359f0637745944944c0fb2ace315e5e07b180445530c7 v1.27.1: d4c46dcc3d210b6eae0b8c34b3ece9f24b1bb2697175615c451db717a99430fb v1.27.0: cf2860aef800496fee0d9fd8722bd7d17c6609e32d87ca380127151f2ce02bb0 + v1.26.13: ab4b851470815c784e67be193d392cfee5d7074b4157b74cc330aef8654ff8f4 v1.26.12: fc942d35f179dc6a3e71dff2ab22fa7a940b2f616f7f933819a3892767cee4f9 v1.26.11: bfa424bccba7c10b4a011ef1cc5b282ac5861e30e3e3897ac8ef1cf8cd95b723 v1.26.10: 5dad57d977923d80f20e1d3b9be5dba806642682a0c1bb511c23543c710761cb diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index 257919ab796..dec7d19cc83 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -16,7 +16,7 @@ kubelet_fail_swap_on: true kubelet_swap_behavior: LimitedSwap ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.28.5 +kube_version: v1.28.6 ## The minimum version working kube_version_min_required: v1.26.0 From df5a06dc70b8ef19aec6b59f5c2c4ab736e5191e Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Sat, 20 Jan 2024 00:29:17 +0800 Subject: [PATCH 031/103] cleanup-for-2.24.0 (#10811) --- .gitlab-ci.yml | 2 +- README.md | 6 +++--- galaxy.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5d17bc923f..45cece0f4dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ stages: - deploy-special variables: - KUBESPRAY_VERSION: v2.23.2 + KUBESPRAY_VERSION: v2.24.0 FAILFASTCI_NAMESPACE: 'kargo-ci' GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray' ANSIBLE_FORCE_COLOR: "true" diff --git a/README.md b/README.md index c3f149db5e4..7ad7e6d58db 100644 --- a/README.md +++ b/README.md @@ -75,11 +75,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou to access the inventory and SSH key in the container, like this: ```ShellSession -git checkout v2.23.2 -docker pull quay.io/kubespray/kubespray:v2.23.2 +git checkout v2.24.0 +docker pull quay.io/kubespray/kubespray:v2.24.0 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ - quay.io/kubespray/kubespray:v2.23.2 bash + quay.io/kubespray/kubespray:v2.24.0 bash # Inside the container you may now run the kubespray playbooks: ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml ``` diff --git a/galaxy.yml b/galaxy.yml index 63731ca141a..33b259d5f7f 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,7 +2,7 @@ namespace: kubernetes_sigs description: Deploy a production ready Kubernetes cluster name: kubespray -version: 2.24.0 +version: 2.25.0 readme: README.md authors: - luksi1 From fe819a6ec38743cb06217765947dd7cb246441ac Mon Sep 17 00:00:00 2001 From: ERIK Date: Mon, 22 Jan 2024 12:56:08 +0800 Subject: [PATCH 032/103] Fix file loss during download (#10779) Signed-off-by: bo.jiang --- contrib/offline/manage-offline-files.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/offline/manage-offline-files.sh b/contrib/offline/manage-offline-files.sh index 875bd5609f9..60c7928b4fd 100755 --- a/contrib/offline/manage-offline-files.sh +++ b/contrib/offline/manage-offline-files.sh @@ -17,7 +17,12 @@ rm -rf "${OFFLINE_FILES_DIR}" rm "${OFFLINE_FILES_ARCHIVE}" mkdir "${OFFLINE_FILES_DIR}" -wget -x -P "${OFFLINE_FILES_DIR}" -i "${FILES_LIST}" +while read -r url; do + if ! wget -x -P "${OFFLINE_FILES_DIR}" "${url}"; then + exit 1 + fi +done < "${FILES_LIST}" + tar -czvf "${OFFLINE_FILES_ARCHIVE}" "${OFFLINE_FILES_DIR_NAME}" [ -n "$NO_HTTP_SERVER" ] && echo "skip to run nginx" && exit 0 From 4cb1f529d19bc9bd8323398cfd7152898ea51aec Mon Sep 17 00:00:00 2001 From: Takuya Murakami <61034+tmurakam@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:39:15 +0900 Subject: [PATCH 033/103] [kubernetes] Add hashes for kubernetes 1.29.0 and 1.29.1 (#10778) * Add hashes of crictl and crio * Add versions of etcd, crictl, crio and csi-snapshotter --- roles/kubespray-defaults/defaults/main/checksums.yml | 12 ++++++++++++ roles/kubespray-defaults/defaults/main/download.yml | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index 4ad16c1acd0..08a0dd5b9fb 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -1,24 +1,28 @@ --- crictl_checksums: arm: + v1.29.0: 0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5 v1.28.0: 1ea267f3872f4b7f311963ab43ce6653ceeaf8727206c889b56587c95497e9dd v1.27.1: ec24fb7e4d45b7f3f3df254b22333839f9bdbde585187a51c93d695abefbf147 v1.27.0: 0b6983195cc62bfc98de1f3fc2ee297a7274fb79ccabf413b8a20765f12d522a v1.26.1: f6b537fd74aed9ccb38be2f49dc9a18859dffb04ed73aba796d3265a1bdb3c57 v1.26.0: 88891ee29eab097ab1ed88d55094e7bf464f3347bc9f056140e45efeddd15b33 arm64: + v1.29.0: 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925 v1.28.0: 06e9224e42bc5e23085751e93cccdac89f7930ba6f7a45b8f8fc70ef663c37c4 v1.27.1: 322bf64d12f9e5cd9540987d47446bf9b0545ceb1900ef93376418083ad88241 v1.27.0: 9317560069ded8e7bf8b9488fdb110d9e62f0fbc0e33ed09fe972768b47752bd v1.26.1: cfa28be524b5da1a6dded455bb497dfead27b1fd089e1161eb008909509be585 v1.26.0: b632ca705a98edc8ad7806f4279feaff956ac83aa109bba8a85ed81e6b900599 amd64: + v1.29.0: d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb v1.28.0: 8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508 v1.27.1: b70e8d7bde8ec6ab77c737b6c69be8cb518ce446365734c6db95f15c74a93ce8 v1.27.0: d335d6e16c309fbc3ff1a29a7e49bb253b5c9b4b030990bf7c6b48687f985cee v1.26.1: 0c1a0f9900c15ee7a55e757bcdc220faca5dd2e1cfc120459ad1f04f08598127 v1.26.0: cda5e2143bf19f6b548110ffba0fe3565e03e8743fadd625fee3d62fc4134eed ppc64le: + v1.29.0: 2803a1865045077f29f798b9c569e1db7d44b5c329a546a0fd183e906925b99f v1.28.0: b70fb7bee5982aa1318ba25088319f1d0d1415567f1f76cd69011b8a14da4daf v1.27.1: c408bb5e797bf02215acf9604c43007bd09cf69353cefa8f20f2c16ab1728a85 v1.27.0: 3e4301c2d4b561d861970004002fe15d49af907963de06c70d326f2af1f145e0 @@ -26,6 +30,8 @@ crictl_checksums: v1.26.0: 5538c88b8ccde419e6158ab9c06dfcca1fa0abecf33d0a75b2d22ceddd283f0d crio_archive_checksums: arm: + v1.29.1: 0 + v1.29.0: 0 v1.28.2: 0 v1.28.1: 0 v1.28.0: 0 @@ -37,6 +43,8 @@ crio_archive_checksums: v1.26.1: 0 v1.26.0: 0 arm64: + v1.29.1: b6d3b502ba1474da370519bcdb9eefa07bb04a5a8e7073fbb7071d6e96f49411 + v1.29.0: 2bf11aeb85362ce4b25a7d9fc17bbe80659013425430e5efb922b4388031a027 v1.28.2: 739923cb744a862039557f23823f4cc12feba121bd26ca3cc01d80cc8aaa1efb v1.28.1: 98a96c6b6bdf20c60e1a7948847c28b57d9e6e47e396b2e405811ea2c24ab9dc v1.28.0: c8ea800244d9e4ce74af85126afadea2939cd6f7ddd152d0f09fafbf294ef1cc @@ -48,6 +56,8 @@ crio_archive_checksums: v1.26.1: 30fe91a60c54b627962da0c21f947424d3cdf484067bc5cda3b3777c10c85384 v1.26.0: 8605b166d00c674e6363ee2336600fa6c6730a6a724f03ab3b72a0d5f9efcd1d amd64: + v1.29.1: 6ff5af9962bfc079275ecec2b6caf5dbd25bf037fff53e87d67f27c2617432c5 + v1.29.0: 79c161d8db8ee7f0f4807d6232283d481ef0c20c514b61289238258f66734ac6 v1.28.2: c8002a622e268b73f8d45b0adbdff9422b832106a23be137fabdc8a233b3f787 v1.28.1: 63cee2e67e283e29d790caa52531bcca7bc59473fb73bde75f4fd8daa169d4bf v1.28.0: fa87497c12815766d18f332b38a4d823fa6ad6bb3d159e383a5557e6c912eb3b @@ -59,6 +69,8 @@ crio_archive_checksums: v1.26.1: cc2fc263f9f88072c744e019ba1c919d9ce2d71603b1b72d288c47c82a86bf08 v1.26.0: 79837d8b7af95547b92dbab105268dd6382ce2a7afbddad93cc168ab0ca766c8 ppc64le: + v1.29.1: c79cc835b66fbf2d2dc8e43908475f0603714f688a1450d4736d1f9f86579b52 + v1.29.0: 8adddaf6cf0ed2905820dc162ca5ef541baa7b251368ee00c75435a872a886fb v1.28.2: 0 v1.28.1: 0 v1.28.0: 0 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 4313391cc22..f609e37cb94 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -134,18 +134,21 @@ skopeo_version: "v1.13.2" kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" etcd_supported_versions: + v1.29: "v3.5.10" v1.28: "v3.5.10" v1.27: "v3.5.10" v1.26: "v3.5.10" etcd_version: "{{ etcd_supported_versions[kube_major_version] }}" crictl_supported_versions: + v1.29: "v1.29.0" v1.28: "v1.28.0" v1.27: "v1.27.1" v1.26: "v1.26.1" crictl_version: "{{ crictl_supported_versions[kube_major_version] }}" crio_supported_versions: + v1.29: v1.29.1 v1.28: v1.28.1 v1.27: v1.27.1 v1.26: v1.26.4 @@ -336,6 +339,7 @@ csi_livenessprobe_image_repo: "{{ kube_image_repo }}/sig-storage/livenessprobe" csi_livenessprobe_image_tag: "v2.5.0" snapshot_controller_supported_versions: + v1.29: "v6.3.3" v1.28: "v4.2.1" v1.27: "v4.2.1" v1.26: "v4.2.1" From a45a40a39888bb144a1caae101d9cbee77d361d1 Mon Sep 17 00:00:00 2001 From: kyrie <139965836+KubeKyrie@users.noreply.github.com> Date: Mon, 22 Jan 2024 21:26:12 +0800 Subject: [PATCH 034/103] update kube-version-min-required to v1.27 (#10817) --- README.md | 2 +- .../defaults/main/checksums.yml | 196 ------------------ .../defaults/main/download.yml | 4 - .../kubespray-defaults/defaults/main/main.yml | 2 +- 4 files changed, 2 insertions(+), 202 deletions(-) diff --git a/README.md b/README.md index 7ad7e6d58db..d69052da15e 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ Note: Upstart/SysV init based OS types are not supported. ## Requirements -- **Minimum required version of Kubernetes is v1.26** +- **Minimum required version of Kubernetes is v1.27** - **Ansible v2.14+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands** - The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/offline-environment.md)) - The target servers are configured to allow **IPv4 forwarding**. diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index 08a0dd5b9fb..d1d18cbe40b 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -5,29 +5,21 @@ crictl_checksums: v1.28.0: 1ea267f3872f4b7f311963ab43ce6653ceeaf8727206c889b56587c95497e9dd v1.27.1: ec24fb7e4d45b7f3f3df254b22333839f9bdbde585187a51c93d695abefbf147 v1.27.0: 0b6983195cc62bfc98de1f3fc2ee297a7274fb79ccabf413b8a20765f12d522a - v1.26.1: f6b537fd74aed9ccb38be2f49dc9a18859dffb04ed73aba796d3265a1bdb3c57 - v1.26.0: 88891ee29eab097ab1ed88d55094e7bf464f3347bc9f056140e45efeddd15b33 arm64: v1.29.0: 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925 v1.28.0: 06e9224e42bc5e23085751e93cccdac89f7930ba6f7a45b8f8fc70ef663c37c4 v1.27.1: 322bf64d12f9e5cd9540987d47446bf9b0545ceb1900ef93376418083ad88241 v1.27.0: 9317560069ded8e7bf8b9488fdb110d9e62f0fbc0e33ed09fe972768b47752bd - v1.26.1: cfa28be524b5da1a6dded455bb497dfead27b1fd089e1161eb008909509be585 - v1.26.0: b632ca705a98edc8ad7806f4279feaff956ac83aa109bba8a85ed81e6b900599 amd64: v1.29.0: d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb v1.28.0: 8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508 v1.27.1: b70e8d7bde8ec6ab77c737b6c69be8cb518ce446365734c6db95f15c74a93ce8 v1.27.0: d335d6e16c309fbc3ff1a29a7e49bb253b5c9b4b030990bf7c6b48687f985cee - v1.26.1: 0c1a0f9900c15ee7a55e757bcdc220faca5dd2e1cfc120459ad1f04f08598127 - v1.26.0: cda5e2143bf19f6b548110ffba0fe3565e03e8743fadd625fee3d62fc4134eed ppc64le: v1.29.0: 2803a1865045077f29f798b9c569e1db7d44b5c329a546a0fd183e906925b99f v1.28.0: b70fb7bee5982aa1318ba25088319f1d0d1415567f1f76cd69011b8a14da4daf v1.27.1: c408bb5e797bf02215acf9604c43007bd09cf69353cefa8f20f2c16ab1728a85 v1.27.0: 3e4301c2d4b561d861970004002fe15d49af907963de06c70d326f2af1f145e0 - v1.26.1: e3026d88722b40deec87711c897df99db3585e2caea17ebd79df5c78f9296583 - v1.26.0: 5538c88b8ccde419e6158ab9c06dfcca1fa0abecf33d0a75b2d22ceddd283f0d crio_archive_checksums: arm: v1.29.1: 0 @@ -37,11 +29,6 @@ crio_archive_checksums: v1.28.0: 0 v1.27.1: 0 v1.27.0: 0 - v1.26.4: 0 - v1.26.3: 0 - v1.26.2: 0 - v1.26.1: 0 - v1.26.0: 0 arm64: v1.29.1: b6d3b502ba1474da370519bcdb9eefa07bb04a5a8e7073fbb7071d6e96f49411 v1.29.0: 2bf11aeb85362ce4b25a7d9fc17bbe80659013425430e5efb922b4388031a027 @@ -50,11 +37,6 @@ crio_archive_checksums: v1.28.0: c8ea800244d9e4ce74af85126afadea2939cd6f7ddd152d0f09fafbf294ef1cc v1.27.1: ddf601e28dc22d878cdd34549402a236afaa47e0a08f39b09e65bab7034b1b97 v1.27.0: c6615360311bff7fdfe1933e8d5030a2e9926b7196c4e7a07fcb10e51a676272 - v1.26.4: dbc64d796eb9055f2e070476bb1f32ab7b7bf42ef0ec23212c51beabfd5ac43f - v1.26.3: c85ea3f6476b354af0b15ad7ab80ae202d082ed0c83f1a323b48352c4698db9a - v1.26.2: 8bd9c912de7f8805c162e089a34ca29e607c48a149940193466ccf7bdf74f606 - v1.26.1: 30fe91a60c54b627962da0c21f947424d3cdf484067bc5cda3b3777c10c85384 - v1.26.0: 8605b166d00c674e6363ee2336600fa6c6730a6a724f03ab3b72a0d5f9efcd1d amd64: v1.29.1: 6ff5af9962bfc079275ecec2b6caf5dbd25bf037fff53e87d67f27c2617432c5 v1.29.0: 79c161d8db8ee7f0f4807d6232283d481ef0c20c514b61289238258f66734ac6 @@ -63,11 +45,6 @@ crio_archive_checksums: v1.28.0: fa87497c12815766d18f332b38a4d823fa6ad6bb3d159e383a5557e6c912eb3b v1.27.1: 23c0b26f9df65671f20c042466c0e6c543e16ba769bbf63aa26abef170f393ba v1.27.0: 8f99db9aeea00299cb3f28ee61646472014cac91930e4c7551c9153f8f720093 - v1.26.4: cfeca97f1ca612813ae0a56a05d33a9f94e3b1fd8df1debb16f322676819314a - v1.26.3: 942772081d9cd4bd0c07e466439b76a1ca95d3f10a7b53dc524d2946b2b17a71 - v1.26.2: 7e030b2e89d4eb2701d9164e67c804fcb872c29accd76f29bcc148a86a920531 - v1.26.1: cc2fc263f9f88072c744e019ba1c919d9ce2d71603b1b72d288c47c82a86bf08 - v1.26.0: 79837d8b7af95547b92dbab105268dd6382ce2a7afbddad93cc168ab0ca766c8 ppc64le: v1.29.1: c79cc835b66fbf2d2dc8e43908475f0603714f688a1450d4736d1f9f86579b52 v1.29.0: 8adddaf6cf0ed2905820dc162ca5ef541baa7b251368ee00c75435a872a886fb @@ -76,11 +53,6 @@ crio_archive_checksums: v1.28.0: 0 v1.27.1: 0 v1.27.0: 0 - v1.26.4: 0 - v1.26.3: 0 - v1.26.2: 0 - v1.26.1: 0 - v1.26.0: 0 # Checksum # Kubernetes versions above Kubespray's current target version are untested and should be used with caution. kubelet_checksums: @@ -104,20 +76,6 @@ kubelet_checksums: v1.27.2: 0 v1.27.1: 0 v1.27.0: 0 - v1.26.13: 74a0e15e47f82e6b073e6fa3eac3ebd2d248bf29a902ee3f0158f204509ca5d6 - v1.26.12: 9bfa6d78d4f6a865d7a1731624d5aafdc69dff54139eed6c2f076158a67c904a - v1.26.11: a7bbe545ab23b068d5e25262bf17a10039edb5c073ba896d036af12afb5fbf9c - v1.26.10: 3130ae14dda009f92296045d9cc2a1f566fa7d85905d0f6a8db47b05496c5d53 - v1.26.9: 739c62a6801d935477121614ee3a2ef6deba78ecd088ae5477c3f18bf19d68c8 - v1.26.8: 9db839028b706c005fb6db4442e7dae32c2916acf826a5666d54236399f447fa - v1.26.7: 85fe65155c5bc0dd851d736003ae94e39c03a3e9d65f7435404d177491b8e08d - v1.26.6: 47d9d6e7a26e70b2f446afaa81a219e14cced8046134023637018e5cf36fa0d8 - v1.26.5: dff080c8fe2b8cce04acc2762452259334b233ac41f557588663daae362db5e2 - v1.26.4: cf78ddc97894d518408bc33ec99e2f4e744d7ab26e598fa6a053b09296c80d00 - v1.26.3: 2c862e06293db71a3644728519818a5448db87347ce5862045e5f3eca6ec13e2 - v1.26.2: 24af93f03e514bb66c6bbacb9f00f393ed57df6e43b2846337518ec0b4d1b717 - v1.26.1: fe940be695f73c03275f049cb17f2bf2eb137014334930ce5c6de12573c1f21f - v1.26.0: cabf702fc542fcbb1173c713f1cbec72fd1d9ded36cdcdbbd05d9c308d8360d1 arm64: v1.29.1: e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d v1.29.0: 0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16 @@ -139,20 +97,6 @@ kubelet_checksums: v1.27.2: 810cd9a611e9f084e57c9ee466e33c324b2228d4249ff38c2588a0cc3224f10d v1.27.1: dbb09d297d924575654db38ed2fc627e35913c2d4000c34613ac6de4995457d0 v1.27.0: 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157 - v1.26.13: 4925b769336df7cf5e7a7f33d82ee5c69ac67a7680ac1f7064534f971d9a79ab - v1.26.12: d4406ed5bfd12768c03fc4fbe011a01e5c91b74d1d4b526fe3ac320d13295ffb - v1.26.11: ff8940394446028e75a2b8155e22eccf635f6a128f45dee41e293493d2743d17 - v1.26.10: ddebcc1af7f203a2ee3d80dad0baaf84a4680748839f5583b39cbce4b8afa7f2 - v1.26.9: f6b1dcee9960ffe6b778dc91cabef8ce4a7bd06c76378ef2784232709eace6a5 - v1.26.8: 0f15e484c4a7a7c3bad9e0aa4d4334ca029b97513fbe03f053201dd937cf316e - v1.26.7: 73e086cfd8cd1cef559e739e19aff2932f8a9e0bdba3c9faeb9185a86d067fbb - v1.26.6: 44c2cd64e1317df8252bca1cf196227c543005a3b10d52fb114401cb1617f32f - v1.26.5: 4256e46eb36bea3c31b0372c4d5b669964a2cfb1eabb7e0e2e0dcb1cdd81f2e8 - v1.26.4: a925a5d20d29c362f0c4d60cb005f21d44576837510e0bc65c817961969b4e7e - v1.26.3: d360f919c279a05441b27178030c3d17134c1f257c95f4b22bdb28c2290993e7 - v1.26.2: 33e77f93d141d3b9e207ae50ff050186dea084ac26f9ec88280f85bab9dad310 - v1.26.1: f4b514162b52d19909cf0ddf0b816d8d7751c5f1de60eda90cd84dcccc56c399 - v1.26.0: fb033c1d079cac8babb04a25abecbc6cc1a2afb53f56ef1d73f8dc3b15b3c09e amd64: v1.29.1: 1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3 v1.29.0: e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05 @@ -174,20 +118,6 @@ kubelet_checksums: v1.27.2: a0d12afcab3b2836de4a427558d067bebdff040e9b306b0512c93d9d2a066579 v1.27.1: cb2845fff0ce41c400489393da73925d28fbee54cfeb7834cd4d11e622cbd3a7 v1.27.0: 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29 - v1.26.13: 05860ef65deb594dc72034c0614f93d7ba9d1f229ff73b43c484298e22465f1b - v1.26.12: aed0a351b01f1e6a84a0992ef1265bb0c9994b900162c075df58d0d02517d3df - v1.26.11: a62953f20fa9fedff50c6c5423e68981e3382d92cf04174d5bca5f4d084de0c5 - v1.26.10: 4c27b3a9f332a6762f7240d0784c64775d4db5a1b881eeae05c4561d06c267ec - v1.26.9: baa2b021ab2f90c342518e2b8981a18de7e1e6b33f11c57e3ff23d40364877a8 - v1.26.8: 1c68a65a6a0c2230325e29da0cc3eaaef9bbf688a7a0bb8243b4a7ebfe0e3363 - v1.26.7: 2926ea2cd7fcd644d24a258bdf21e1a8cfd95412b1079914ca46466dae1d74f2 - v1.26.6: da82477404414eb342d6b93533f372aa1c41956a57517453ef3d39ebbfdf8cc2 - v1.26.5: ad5e318ff0e81bc2bef874b2038489722cfcc117bd31726d0193056458c18bff - v1.26.4: 1e29fe7a097066cfbc1c1d2ab37f8b883c8f3fec414bafe8f2c7b960b0fb60fe - v1.26.3: 992d6298bd494b65f54c838419773c4976aca72dfb36271c613537efae7ab7d2 - v1.26.2: e6dd2ee432a093492936ff8505f084b5ed41662f50231f1c11ae08ee8582a3f5 - v1.26.1: 8b99dd73f309ca1ac4005db638e82f949ffcfb877a060089ec0e729503db8198 - v1.26.0: b64949fe696c77565edbe4100a315b6bf8f0e2325daeb762f7e865f16a6e54b5 ppc64le: v1.29.1: 467d2b457205363f53f72081295ea390fc25215b0ccc29dc04c4f82925266067 v1.29.0: 67f09f866d3e4aee8211ce9887ec8bc427b188474a882a7af999fc0fee939028 @@ -209,20 +139,6 @@ kubelet_checksums: v1.27.2: 3af92edd687f7932e7fce877944dfe5efa437bf5f171fc8331725c631a1a86ef v1.27.1: 7a800b9539beaba0b5d6357070a40fb3c4d216c2ad6693b15f9b1307b1c99e1f v1.27.0: 17c061a9f7919697ac71c151c19337f65b86f59f59441687ac92e977d851c75b - v1.26.13: 5ebb3cf365b4b5a70de859cb8a5b5c6abb359fa4ffe101a5f960c99f2342e79e - v1.26.12: c740ea116206a3e9420b1ab810991c21910abeea947da1b295c2454700cdbe44 - v1.26.11: 408e47e33159551ebeeb6f4ef836bd7d07fc92d16ca6d662c85ea8fa0fe34a8e - v1.26.10: 5cc1046c8f9188dc6d5ec22200eeed7c8c1ae88a23ce5d79e8dfaf1ff61f2bf8 - v1.26.9: e87a83c1ca74e55cea51eda53d29324de7fb7f9330c266ea1f2e270fe0f9b677 - v1.26.8: 92c8deba1f6a89a6d6555c224cebab43d141d5822c252511988ad43ff1a7cc1d - v1.26.7: db5d946bad409a1cea177564fb4111e03e4efc15e86d0078fee401022a4b057a - v1.26.6: 1ca83394b04d3017803a30671eb699a61201e00b656e1fc5b833bd83f8835ff4 - v1.26.5: 787a27855228760a6eeeb200a0e7eab82cb7603b0045ddbadcc1a24f9dc2f178 - v1.26.4: a0d653ed1f5f90d380edb5d6ff77ff61e39e8f1a39dd68719c0126ef6f19c381 - v1.26.3: a12a78b68ec8ac76d482d8a95e0d927ffedac62e630af5fef704f5fecf8e92d9 - v1.26.2: 6f03bc34a34856a3e0e667bea8d6817596c53c7093482d44516163639ce39625 - v1.26.1: bf795bec9b01a9497f46f47b2f3466628fba11f7834c9b2a0f3fa1028c061144 - v1.26.0: df13099611f4eada791e5f41948ef94f7b52c468dff1a6fcad54f4c56b467a7f kubectl_checksums: arm: v1.29.1: a4b478cc0e9adaab0c5bb3627c20c5228ea0fe2aeff9e805d611eb3edb761972 @@ -245,20 +161,6 @@ kubectl_checksums: v1.27.2: 7792f5630543c0af84f444521ee6113da5ae00f2b50872d57324aa725a5341c5 v1.27.1: fe704e355bf2c5f69964cd12772687535a11a5e9ec0baf4f27e0a8fb156bc615 v1.27.0: 288470e3eb89a2f55273d753ce6674dfb00e732f2971428acb964810aa726188 - v1.26.13: 1a073f6f50e4afae000e286200e8face5c5e8b8545d8f70b163d4a2431ab096a - v1.26.12: cf788244170a71dc6540abccdff76327a48d8c5fbc88ffde9e1ebaa4616f8d2e - v1.26.11: 34cc371daef6e42aea27117aa643619ab870d1cb61995530fc9bcb992e3caf10 - v1.26.10: d26ab68b4884eec4dcadd0f564ba197d48f21aebed509b9b1541d31f2aaf5890 - v1.26.9: 8e020ffe72dd4c8694ee5e9f124833ca302a2341fa046650482b38ddb189d1fd - v1.26.8: 411c5c6ba9a247d7fa30f68fd37cfdb92ef14326127bed2512a0daf11a6097d4 - v1.26.7: ad796f714102a78a4f4dfa8f49a3c11cb31a9d74965d6b14f84ef5adb065ed69 - v1.26.6: 1d67bdd384d382decb5b61a3c28d5b05ef0251296551bbc46102c02518c41c02 - v1.26.5: eb3e9bac15ebd31f7e4c21782f60ce9285f59ad4072a78bb53b27029b33609b6 - v1.26.4: 69fccd21b6f7a27d96cda7e6e0cfd3741e3a5bcd7348f2f6e2e9c7550809f030 - v1.26.3: cdb3f670396a775119eb84436d6c0e7e29f24ec511681049200eeb39df9960fb - v1.26.2: a8944021fc9022f73976d8ab2736f21b64b30de3b2a6ccfddd0316ca1d3c6a1d - v1.26.1: e067d59ac19e287026b5c2b75a1077b1312ba82ad64ee01dff2cdafd57720f39 - v1.26.0: 8eef310d0de238c582556d81ab8cbe8d6fca3c0e43ee337a905dcdd3578f9dda arm64: v1.29.1: 96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b v1.29.0: 8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7 @@ -280,20 +182,6 @@ kubectl_checksums: v1.27.2: 1b0966692e398efe71fe59f913eaec44ffd4468cc1acd00bf91c29fa8ff8f578 v1.27.1: fd3cb8f16e6ed8aee9955b76e3027ac423b6d1cc7356867310d128082e2db916 v1.27.0: f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc - v1.26.13: 4abebc34c114111b81bbf5222f0810e6899937d04bc453d9ccd77046643bbcda - v1.26.12: b9cfdb6c9a53146330d5694e711c48febb7ec022aea8d16a51b5e12d761580cc - v1.26.11: 4a6d2b7204af3cf84cd0e2c670fbb211501050c9a288de49de3c6363d4e0a63e - v1.26.10: 5752e3908fa1d338eb1fa99a6f39c6a4c27b065cb459da84e35c4ec718879f14 - v1.26.9: f945c63220b393ddf8df67d87e67ff74b7f56219a670dee38bc597a078588e90 - v1.26.8: e93f836cba409b5ef5341020d9501067a51bf8210cb35649518e5f4d114244cf - v1.26.7: 71edc4c6838a7332e5f82abb35642ce7f905059a258690b0a585d3ed6de285b3 - v1.26.6: 8261d35cd374c438104bb5257e6c9dafb8443cd0eed8272b219ec5aa17b8ca40 - v1.26.5: c3b3de6a2d7f7e1902c65f6774754e62e86d464ed259509ba29a2a209a515ddf - v1.26.4: eea4054825a4c20cc09bc15abcb1354725ad886338e6892141a071caab91d4b6 - v1.26.3: 0f62cbb6fafa109f235a08348d74499a57bb294c2a2e6ee34be1fa83432fec1d - v1.26.2: 291e85bef77e8440205c873686e9938d7f87c0534e9a491de64e3cc0584295b6 - v1.26.1: 4027cb0a2840bc14ec3f18151b3360dd2d1f6ce730ed5ac28bd846c17e7d73f5 - v1.26.0: 79b14e4ddada9e81d2989f36a89faa9e56f8abe6e0246e7bdc305c93c3731ea4 amd64: v1.29.1: 69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f v1.29.0: 0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5 @@ -315,20 +203,6 @@ kubectl_checksums: v1.27.2: 4f38ee903f35b300d3b005a9c6bfb9a46a57f92e89ae602ef9c129b91dc6c5a5 v1.27.1: 7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d v1.27.0: 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87 - v1.26.13: e4bad4273431f9f5f05f27f5c2054cbbad6d9ee00b85e0810cb4ef0489b02571 - v1.26.12: 8e6af8d68e7b9d2a1eb43255c0da793276e549a34a2b9c3c87a9c26438e7fd71 - v1.26.11: 27c34a0870230d9dd723e1e01114634e396cd2a3d25ced263b769a4bd53e4edd - v1.26.10: 93ad44b4072669237247bfbc171be816f08e7e9e4260418d2cfdd0da1704ae86 - v1.26.9: 98ea4a13895e54ba24f57e0d369ff6be0d3906895305d5390197069b1da12ae2 - v1.26.8: d8e0dba258d1096f95bb6746ca359db2ee8abe226e777f89dc8a5d1bb76795aa - v1.26.7: d9dc7741e5f279c28ef32fbbe1daa8ebc36622391c33470efed5eb8426959971 - v1.26.6: ee23a539b5600bba9d6a404c6d4ea02af3abee92ad572f1b003d6f5a30c6f8ab - v1.26.5: 5080bb2e9631fe095139f7e973df9a31eb73e668d1785ffeb524832aed8f87c3 - v1.26.4: 636ac0eaa467dbceda4b2c4e33662adc9709f5ce40341c9fc1a687fc276ac02d - v1.26.3: 026c8412d373064ab0359ed0d1a25c975e9ce803a093d76c8b30c5996ad73e75 - v1.26.2: fcf86d21fb1a49b012bce7845cf00081d2dd7a59f424b28621799deceb5227b3 - v1.26.1: d57be22cfa25f7427cfb538cfc8853d763878f8b36c76ce93830f6f2d67c6e5d - v1.26.0: b6769d8ac6a0ed0f13b307d289dc092ad86180b08f5b5044af152808c04950ae ppc64le: v1.29.1: b7780124ccfe9640f3a37d242d31e8dbb252bcd379bd0d7bf3776d15baf15ca3 v1.29.0: ea926d8cf25e2ce982ff5c375da32b51ccbd122b721b1bc4a32f52a9a0d073ab @@ -350,20 +224,6 @@ kubectl_checksums: v1.27.2: efee037a276f72c77cc230194d7dadf943a5778be46b7985edeb414d27894266 v1.27.1: 440bcfd9611319f3d9e5d4fa4cdee2421cdf80c01fad223934d9a9b640673d75 v1.27.0: daa9f1d4fe3f217de2546bca4ac14601f34b34a25c1f571f1e44eb313aee1385 - v1.26.13: d1d9058ca73bf669d4706371a5af9340384756392608a2f264b37977d62b1efb - v1.26.12: f977850aff89f7e148fbe940aa84faf7e1abbf1936527c79fc111a6f759b73b0 - v1.26.11: f2c577e75277415203e699ca7bc08eb640a0b8211a0366926122e3ac4f98997c - v1.26.10: 24674d7ce1ccef207d127bd952f6f40f951d7682cfa603f6989ed80e91dcaafc - v1.26.9: bcb287f24a30bd7ef27bc36dc4f896aba3f1091f947afde73576fbd81af65cc5 - v1.26.8: e94748f8954f44bd5ad5be78a2906ee6a8db7c00ea2d50c9db1bfa09cfc097b9 - v1.26.7: 307eabd20201d1a8f9ac433c03716333565c6bd2532dce4bb42eddc88458d509 - v1.26.6: b56f4422fcf0dc095e777d29eb7eb18cf080098ea47ffdd0a1797a0f8e897fac - v1.26.5: 85f18cad385df01f1758d17c0a0b7f865288121dcc64229a07abb32279b0e44b - v1.26.4: 4f5686ea674d37a639389d95c2a32661986f6f06b530076da2b178839b213414 - v1.26.3: dbfc55dcb86e3e7a2ca01df0317d27b8026861d472bcc7bffa33f45dee693927 - v1.26.2: 88ab57859ac8c4559834d901884ed906c88ea4fa41191e576b34b6c8bd6b7a16 - v1.26.1: 5cfd9fea8dea939a2bd914e1caa7829aa537702ddf14e02a59bf451146942fde - v1.26.0: 9e2b2a03ee5fc726ebd39b6a09498b6508ade8831262859c280d39df33f8830d kubeadm_checksums: arm: v1.29.1: 0 @@ -386,20 +246,6 @@ kubeadm_checksums: v1.27.2: 0 v1.27.1: 0 v1.27.0: 0 - v1.26.13: 2b61d87e7babc764ac53783af300dffef556f1ea9039b9ac550f23c6048ee1e1 - v1.26.12: a15b212a2ec393b229d35eb0900b287c4a4c52e1993ef69eadcce1105d5c2bc9 - v1.26.11: 3683f52861f89c3cf48ea7754fd2e81ad30a51bef57d056b2663e5e9139870b6 - v1.26.10: b59433ebaddf5c5d816033304989f9dd2e0b863650532cd195c1d41c0f973f14 - v1.26.9: a6841e7e554407776e4d0fc83306756ad1836d1f92d6d5cce1055eee1999732a - v1.26.8: 31f37eeed5a9e23719e97055051a5efada2fb69deda958056b3d6b0b41e7eaa5 - v1.26.7: bcff0b0a94f6ee6665a0b1eff0f6aa15ca6caac5040cbbf79cd9bd1125088a5d - v1.26.6: 345775290b5379ab24c6cb333bc26d9bc934a1ce8b795c2948d28608b9d6cb9c - v1.26.5: 7915c12580524fe9e15e35849c77c7a6981849b26ea9324b21efc7b2d66727d1 - v1.26.4: 5517d64c030bc48211b9025b8b70bee1430cc278c3e1dc520967b189f6fa66f9 - v1.26.3: 7ceaf94361e6e7a9f877388df30e424604504de4e36f24aafe30f31f2a27600c - v1.26.2: 84982a359967571baf572aa1a61397906f987d989854ebb67a03b09ea4502af4 - v1.26.1: 0dbd0a197013a3fdc5cb3e012fa8b0d50f38fd3dda56254f4648e08ac867fb60 - v1.26.0: 3368537a5e78fdbfa3cbcae0d19102372a0f4eb6b6a78e7b6b187d5db86d6c9e arm64: v1.29.1: 3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf v1.29.0: bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980 @@ -421,20 +267,6 @@ kubeadm_checksums: v1.27.2: 8f01f363f7c7f92de2f2276124a895503cdc5a60ff549440170880f296b087eb v1.27.1: 024a59cd6fc76784b597c0c1cf300526e856e8c9fefa5fa7948158929b739551 v1.27.0: acd805c6783b678ee0068b9dd8165bbfd879c345fd9c25d6a978dbc965f48544 - v1.26.13: 2cce047abb5df5c9e1e48405e1d4822f2ccc6609668db49e95d9ccbbeaa71a7a - v1.26.12: 2dee03d460e8a1b3f30e5ef48ce94f5eeb4ff4550e65860e6e2d94b368c2cde6 - v1.26.11: a13318c1493e58a9f7c4359c79443f3c86a690ec601bcc76308c809d8d61edb8 - v1.26.10: 1ddcb47ee4f7171736dbacc046a7ceae55411ee09920435c3821b530f4650428 - v1.26.9: 14c87cbb9a3fa02308a9546aad192ce2d93e5d1d0296d28ba449079e6a1cb2b2 - v1.26.8: f12d5d748abb8586723b78a2f0300f88faf0391f56d4d49f1ad1cef74160a1b5 - v1.26.7: 34192ceac2287029b36e2d6b682e55dee245ae622701dc3b36bd3203019b18d1 - v1.26.6: 003c7740750ad92d2ff3d58d4a15015906c120c93c7aa605ba98edd936061542 - v1.26.5: d7eede9b44850e16cbe4bb8946a79c03c2c0272f7adc726e63b3a1ac09f13b55 - v1.26.4: a97052d393e60027c354e97c88493aa14a76c8cfb7418bbdf8425b3711d86e3a - v1.26.3: e9a7dbca77f9576a98af1db8747e9dc13e930e40295eaa259dd99fd6e17a173f - v1.26.2: f210d8617acf7c601196294f7ca97e4330b75dad00df6b8dd12393730c501473 - v1.26.1: db101c4bb8e33bd69241de227ed317feee6d44dbd674891e1b9e11c6e8b369bb - v1.26.0: 652844c9518786273e094825b74a1988c871552dc6ccf71366558e67409859d1 amd64: v1.29.1: d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd v1.29.0: 629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c @@ -456,20 +288,6 @@ kubeadm_checksums: v1.27.2: 95c4bfb7929900506a42de4d92280f06efe6b47e0a32cbc1f5a1ed737592977a v1.27.1: c7d32d698e99b90f877025104cb4a9f3f8c707e99e6817940f260135b6d1ad0a v1.27.0: 78d0e04705a7bdb76a514d60f60c073b16334b15f57ee87f064354ca8a233e80 - v1.26.13: 62c0e49f749d230f0c27fa53cbe4851894d9002757fd3653fc5da36aa7c68032 - v1.26.12: 5a5d65acefb50010859be8ffba8e6e059d552ae357e3101c12c62e747a9416a2 - v1.26.11: 58f886e39e517ba1a92493f136e80f1b6ea9362966ad9d2accdf2133004161f2 - v1.26.10: 27ed1d857f4a315f3d059168c6e25fdbf0559f9c8e59bab6c50e7921f74dadbf - v1.26.9: 73e128821dd1f799a75c922218d12f6c4618b8e29cc7dae2a7390fb80092d3d9 - v1.26.8: 233a89277ca49dbd666b7391c6c0e43c33d2f08052d5b93e9cd0100ee69430c8 - v1.26.7: 812e6d0e94a3fc77d3e9d09dbe709190b77408936cc4e960d916e8401be11090 - v1.26.6: ba699c3c26aaf64ef46d34621de9f3b62e37656943e09f23dc3bf5aa7b3f5094 - v1.26.5: 793767419c382bae2dc2c9396baafbf051bfa3214accf40dcd7c5ea405583802 - v1.26.4: aa1a137aa2c3427f199ff652c96b11d6b124358296996eb7b8cbde220607b2fe - v1.26.3: 87a1bf6603e252a8fa46be44382ea218cb8e4f066874d149dc589d0f3a405fed - v1.26.2: 277d880dc6d79994fd333e49d42943b7c9183b1c4ffdbf9da59f806acec7fd82 - v1.26.1: 1531abfe96e2e9d8af9219192c65d04df8507a46a081ae1e101478e95d2b63da - v1.26.0: 72631449f26b7203701a1b99f6914f31859583a0e247c3ac0f6aaf59ca80af19 ppc64le: v1.29.1: 3ec6d90c05dd8e4c6bb1f42fd2fe0f091d85317efaf47d9baebd9af506b3878b v1.29.0: 4c414a463ed4277e9062c797d1c0435aa7aec2fd1688c5d34e3161c898113cb5 @@ -491,20 +309,6 @@ kubeadm_checksums: v1.27.2: 412bccd310f4976201d359f0637745944944c0fb2ace315e5e07b180445530c7 v1.27.1: d4c46dcc3d210b6eae0b8c34b3ece9f24b1bb2697175615c451db717a99430fb v1.27.0: cf2860aef800496fee0d9fd8722bd7d17c6609e32d87ca380127151f2ce02bb0 - v1.26.13: ab4b851470815c784e67be193d392cfee5d7074b4157b74cc330aef8654ff8f4 - v1.26.12: fc942d35f179dc6a3e71dff2ab22fa7a940b2f616f7f933819a3892767cee4f9 - v1.26.11: bfa424bccba7c10b4a011ef1cc5b282ac5861e30e3e3897ac8ef1cf8cd95b723 - v1.26.10: 5dad57d977923d80f20e1d3b9be5dba806642682a0c1bb511c23543c710761cb - v1.26.9: 1cd0e3623b93aa1786dddb73570a841323db35df4eca45004db2046550ca5d12 - v1.26.8: c93248ce2c9906d16fcb7590d8f3929406b28967da79d6a01c2b2d39203a7f58 - v1.26.7: f8d35a26349c28a01244cfb2f0a163c11daa6bd501e64ce261455c38ffd29bc5 - v1.26.6: d09b5e2221c26f47e2f048b0c375540db14090a4d7b71a708fe51a2b3d0e2b81 - v1.26.5: d6296ca1be9ab7914e9fcd770ce46184db41f5613cec5b1b3de9d51439052fba - v1.26.4: f573ce081e884cc642750f8915d3fdf0ce5696c0d5b4f918d0ff20e76e482739 - v1.26.3: 80b00286e54a87645908c7fd284caef0b1cd7fab5a1948518a6a8d6b0852d49d - v1.26.2: f5d610c4a8a4f99ac6dd07f8cbc0db1de602d5a8895cdaa282c72e36183e310b - v1.26.1: 89ad4d60d266e32147c51e7fb972a9aa6c382391822fa00e27a20f769f3586e8 - v1.26.0: 2431061b3980caa9950a9faaafdfb5cd641e0f787d381db5d10737c03ad800c6 etcd_binary_checksums: arm: v3.5.10: 0 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index f609e37cb94..941e04da077 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -137,21 +137,18 @@ etcd_supported_versions: v1.29: "v3.5.10" v1.28: "v3.5.10" v1.27: "v3.5.10" - v1.26: "v3.5.10" etcd_version: "{{ etcd_supported_versions[kube_major_version] }}" crictl_supported_versions: v1.29: "v1.29.0" v1.28: "v1.28.0" v1.27: "v1.27.1" - v1.26: "v1.26.1" crictl_version: "{{ crictl_supported_versions[kube_major_version] }}" crio_supported_versions: v1.29: v1.29.1 v1.28: v1.28.1 v1.27: v1.27.1 - v1.26: v1.26.4 crio_version: "{{ crio_supported_versions[kube_major_version] }}" yq_version: "v4.35.2" @@ -342,7 +339,6 @@ snapshot_controller_supported_versions: v1.29: "v6.3.3" v1.28: "v4.2.1" v1.27: "v4.2.1" - v1.26: "v4.2.1" snapshot_controller_image_repo: "{{ kube_image_repo }}/sig-storage/snapshot-controller" snapshot_controller_image_tag: "{{ snapshot_controller_supported_versions[kube_major_version] }}" diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index dec7d19cc83..cbad203e493 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -19,7 +19,7 @@ kubelet_swap_behavior: LimitedSwap kube_version: v1.28.6 ## The minimum version working -kube_version_min_required: v1.26.0 +kube_version_min_required: v1.27.0 ## Kube Proxy mode One of ['iptables', 'ipvs'] kube_proxy_mode: ipvs From 3e7b568d3eb485902dcaa819a7f15ba288dd3abd Mon Sep 17 00:00:00 2001 From: Noam Date: Mon, 22 Jan 2024 18:11:08 +0200 Subject: [PATCH 035/103] crictl allow setting grace period for stop containers upon reset (#10651) * crictl allow setting different grace period for stop containers and pods * correct grace period location --- roles/reset/defaults/main.yml | 3 +++ roles/reset/tasks/main.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/reset/defaults/main.yml b/roles/reset/defaults/main.yml index 3d58dd91b52..b63f2e2a85d 100644 --- a/roles/reset/defaults/main.yml +++ b/roles/reset/defaults/main.yml @@ -16,3 +16,6 @@ reset_restart_network_service_name: >- {%- elif ansible_os_family == "Debian" -%} networking {%- endif %} + +# crictl stop container grace period +cri_stop_containers_grace_period: 0 diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 84be011a516..a0fa4093c0f 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -52,7 +52,7 @@ register: crictl - name: Reset | stop all cri containers - shell: "set -o pipefail && {{ bin_dir }}/crictl ps -q | xargs -r {{ bin_dir }}/crictl -t 60s stop" + shell: "set -o pipefail && {{ bin_dir }}/crictl ps -q | xargs -r {{ bin_dir }}/crictl -t 60s stop -t {{ cri_stop_containers_grace_period }}" args: executable: /bin/bash register: remove_all_cri_containers From 4e52fb7a1f85e04a1d004e8c55a8f21db8cb57c7 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 22 Jan 2024 17:11:17 +0100 Subject: [PATCH 036/103] Adjust the releases process documentation. (#10727) * markdown numbered lists: no need to manually maitain the numbers * add instructions to update the collection version roughly following Ansible collection maintenance guidelines (https://docs.ansible.com/ansible/latest/community/collection_contributors/collection_release_with_branches.html) * Distinguish major from minor releases in the release process * Bump current galaxy.yml version to the next release --- RELEASE.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 296040de1e2..8248a097bc9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -3,17 +3,19 @@ The Kubespray Project is released on an as-needed basis. The process is as follows: 1. An issue is proposing a new release with a changelog since the last release. Please see [a good sample issue](https://github.com/kubernetes-sigs/kubespray/issues/8325) -2. At least one of the [approvers](OWNERS_ALIASES) must approve this release -3. The `kube_version_min_required` variable is set to `n-1` -4. Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables. -5. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details. -6. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes -7. An approver creates a release branch in the form `release-X.Y` -8. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details. -9. The `KUBESPRAY_VERSION` variable is updated in `.gitlab-ci.yml` -10. The release issue is closed -11. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released` -12. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...` +1. At least one of the [approvers](OWNERS_ALIASES) must approve this release +1. (Only for major releases) The `kube_version_min_required` variable is set to `n-1` +1. (Only for major releases) Remove hashes for [EOL versions](https://github.com/kubernetes/website/blob/main/content/en/releases/patch-releases.md) of kubernetes from `*_checksums` variables. +1. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details. +1. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes +1. (Only for major releases) An approver creates a release branch in the form `release-X.Y` +1. (For major releases) On the `master` branch: bump the version in `galaxy.yml` to the next expected major release (X.y.0 with y = Y + 1), make a Pull Request. +1. (For minor releases) On the `release-X.Y` branch: bump the version in `galaxy.yml` to the next expected minor release (X.Y.z with z = Z + 1), make a Pull Request. +1. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details. +1. (Only for major releases) The `KUBESPRAY_VERSION` in `.gitlab-ci.yml` is upgraded to the version we just released # TODO clarify this, this variable is for testing upgrades. +1. The release issue is closed +1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released` +1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...` ## Major/minor releases and milestones From 0e971a37aa1c50a58975b615870c313590eebfcd Mon Sep 17 00:00:00 2001 From: Yuhao Zhang Date: Mon, 22 Jan 2024 08:22:27 -0800 Subject: [PATCH 037/103] Offline control plane recover (#10660) * ignore_unreachable for etcd dir cleanup ignore_errors ignores errors occur within "file" module. However, when the target node is offline, the playbook will still fail at this task with node "unreachable" state. Setting "ignore_unreachable: true" allows the playbook to bypass offline nodes and move on to proceed recovery tasks on remaining online nodes. * Re-arrange control plane recovery runbook steps * Remove suggestion to manually update IP addresses The suggestion was added in 48a182844c9c3438e36c78cbc4518c962e0a9ab2 4 years ago. But a new task added 2 years ago, in ee0f1e9d58ed8bf1fd13ff1eb1527678fe4fa6da, automatically update API server arg with updated etcd node ip addresses. This suggestion is no longer needed. --- docs/recover-control-plane.md | 10 ++++------ roles/recover_control_plane/etcd/tasks/main.yml | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/recover-control-plane.md b/docs/recover-control-plane.md index 0b80da271dc..9174789cfa0 100644 --- a/docs/recover-control-plane.md +++ b/docs/recover-control-plane.md @@ -3,11 +3,6 @@ To recover from broken nodes in the control plane use the "recover\-control\-plane.yml" playbook. -* Backup what you can -* Provision new nodes to replace the broken ones -* Place the surviving nodes of the control plane first in the "etcd" and "kube\_control\_plane" groups -* Add the new nodes below the surviving control plane nodes in the "etcd" and "kube\_control\_plane" groups - Examples of what broken means in this context: * One or more bare metal node(s) suffer from unrecoverable hardware failure @@ -19,8 +14,12 @@ __Note that you need at least one functional node to be able to recover using th ## Runbook +* Backup what you can +* Provision new nodes to replace the broken ones * Move any broken etcd nodes into the "broken\_etcd" group, make sure the "etcd\_member\_name" variable is set. * Move any broken control plane nodes into the "broken\_kube\_control\_plane" group. +* Place the surviving nodes of the control plane first in the "etcd" and "kube\_control\_plane" groups +* Add the new nodes below the surviving control plane nodes in the "etcd" and "kube\_control\_plane" groups Then run the playbook with ```--limit etcd,kube_control_plane``` and increase the number of ETCD retries by setting ```-e etcd_retries=10``` or something even larger. The amount of retries required is difficult to predict. @@ -35,7 +34,6 @@ The playbook attempts to figure out it the etcd quorum is intact. If quorum is l ## Caveats * The playbook has only been tested with fairly small etcd databases. -* If your new control plane nodes have new ip addresses you may have to change settings in various places. * There may be disruptions while running the playbook. * There are absolutely no guarantees. diff --git a/roles/recover_control_plane/etcd/tasks/main.yml b/roles/recover_control_plane/etcd/tasks/main.yml index 66dbc8b6dec..599f56b1506 100644 --- a/roles/recover_control_plane/etcd/tasks/main.yml +++ b/roles/recover_control_plane/etcd/tasks/main.yml @@ -39,6 +39,7 @@ delegate_to: "{{ item }}" with_items: "{{ groups['broken_etcd'] }}" ignore_errors: true # noqa ignore-errors + ignore_unreachable: true when: - groups['broken_etcd'] - has_quorum From fd6bb0f7fd30183daa0195c7b70fb22becd3ac93 Mon Sep 17 00:00:00 2001 From: Kri5 Date: Mon, 22 Jan 2024 17:22:37 +0100 Subject: [PATCH 038/103] Allow vagrant directory location to be configurable (#10718) In the case were vagrant is not invoked directly from the repository, but from another location, and the Vagrantfile is "included" into another, we need to be able to specify where the location of the vagrant directory is, as of now it's hardcoded relative to the Vagrantfile location. This commit fix it. --- Vagrantfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 49873766872..fb8f1a7179c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -77,6 +77,8 @@ $libvirt_nested ||= false $ansible_verbosity ||= false $ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || "" +$vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant") + $playbook ||= "cluster.yml" host_vars = {} @@ -96,7 +98,7 @@ $inventory = File.absolute_path($inventory, File.dirname(__FILE__)) # if $inventory has a hosts.ini file use it, otherwise copy over # vars etc to where vagrant expects dynamic inventory to be if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini")) - $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant", "provisioners", "ansible") + $vagrant_ansible = File.join(File.absolute_path($vagrant_dir), "provisioners", "ansible") FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible) $vagrant_inventory = File.join($vagrant_ansible,"inventory") FileUtils.rm_f($vagrant_inventory) From 5a405336aee052a4c2304aba9dd17194cb6eac4f Mon Sep 17 00:00:00 2001 From: my-git9 Date: Tue, 23 Jan 2024 00:28:09 +0800 Subject: [PATCH 039/103] Support following k8s version selection pause image (#10756) Signed-off-by: xin.li --- roles/kubespray-defaults/defaults/main/download.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 941e04da077..676989ea310 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -115,7 +115,6 @@ flannel_version: "v0.22.0" flannel_cni_version: "v1.1.2" cni_version: "v1.3.0" weave_version: 2.8.1 -pod_infra_version: "3.9" cilium_version: "v1.13.4" cilium_cli_version: "v0.15.0" @@ -133,6 +132,12 @@ skopeo_version: "v1.13.2" # Get kubernetes major version (i.e. 1.17.4 => 1.17) kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" +pod_infra_supported_versions: + v1.28: "3.9" + v1.27: "3.9" + v1.26: "3.9" +pod_infra_version: "{{ pod_infra_supported_versions[kube_major_version] }}" + etcd_supported_versions: v1.29: "v3.5.10" v1.28: "v3.5.10" From 54fb75f0e085fb81a5b1709e0d61ff735d8a28c1 Mon Sep 17 00:00:00 2001 From: "R. P. Taylor" <1686627+rptaylor@users.noreply.github.com> Date: Mon, 22 Jan 2024 08:28:19 -0800 Subject: [PATCH 040/103] don't overwrite changes to openstack allowed_address_pairs (#10760) --- .../openstack/modules/compute/main.tf | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 64ccc7ff075..d015636f0f1 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -304,6 +304,10 @@ resource "openstack_networking_port_v2" "k8s_master_port" { } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] @@ -370,6 +374,10 @@ resource "openstack_networking_port_v2" "k8s_masters_port" { } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] @@ -434,6 +442,10 @@ resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" { } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] @@ -560,6 +572,10 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_port" { } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] @@ -620,6 +636,10 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_no_etcd_port" } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] @@ -681,6 +701,10 @@ resource "openstack_networking_port_v2" "k8s_node_port" { } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] @@ -747,6 +771,10 @@ resource "openstack_networking_port_v2" "k8s_node_no_floating_ip_port" { } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] @@ -808,6 +836,10 @@ resource "openstack_networking_port_v2" "k8s_nodes_port" { } } + lifecycle { + ignore_changes = [ allowed_address_pairs ] + } + depends_on = [ var.network_router_id ] From 6497ecc767a3b1eeec5c6c4be8872dceb7a77774 Mon Sep 17 00:00:00 2001 From: "R. P. Taylor" <1686627+rptaylor@users.noreply.github.com> Date: Mon, 22 Jan 2024 08:28:28 -0800 Subject: [PATCH 041/103] workaround for terraform bug related to no_floating and extra_groups (#10764) --- contrib/terraform/openstack/modules/compute/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index d015636f0f1..d161d26f627 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -883,7 +883,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" { metadata = { ssh_user = var.ssh_user - kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}" + kubespray_groups = "kube_node,k8s_cluster,%{if !each.value.floating_ip}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}" depends_on = var.network_router_id use_access_ip = var.use_access_ip } From a2ed5fcd3d00cae4ac4ff6c525ffd74f7a5385bb Mon Sep 17 00:00:00 2001 From: Serge Hartmann Date: Mon, 22 Jan 2024 17:49:00 +0100 Subject: [PATCH 042/103] Doc: variable cilium_ipsec_key must be base64 encoded (#10781) Signed-off-by: serge Hartmann --- docs/cilium.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/cilium.md b/docs/cilium.md index 0482b0b1885..a773dbc902f 100644 --- a/docs/cilium.md +++ b/docs/cilium.md @@ -141,7 +141,7 @@ cilium_encryption_enabled: true cilium_encryption_type: "ipsec" ``` -The third variable is `cilium_ipsec_key.` You need to create a secret key string for this variable. +The third variable is `cilium_ipsec_key`. You need to create a secret key string for this variable. Kubespray does not automate this process. Cilium documentation currently recommends creating a key using the following command: @@ -149,7 +149,11 @@ Cilium documentation currently recommends creating a key using the following com echo "3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128" ``` -Note that Kubespray handles secret creation. So you only need to pass the key as the `cilium_ipsec_key` variable. +Note that Kubespray handles secret creation. So you only need to pass the key as the `cilium_ipsec_key` variable, base64 encoded: + +```shell +echo "cilium_ipsec_key: "$(echo -n "3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128" | base64 -w0) +``` ### Wireguard Encryption From 22bb0976d5a4ea844f5a2e37e855a57ea705bd43 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 22 Jan 2024 17:49:14 +0100 Subject: [PATCH 043/103] Adjust kubelet_event_record_qps to K8S default (#10826) Also remove redundant check in the kubelet config template (we define a default, so the setting will always be "true") --- .../kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 | 2 -- roles/kubespray-defaults/defaults/main/main.yml | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index f54d1f8b3ee..1cd00992af7 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -135,9 +135,7 @@ tlsCipherSuites: - {{ tls }} {% endfor %} {% endif %} -{% if kubelet_event_record_qps %} eventRecordQPS: {{ kubelet_event_record_qps }} -{% endif %} shutdownGracePeriod: {{ kubelet_shutdown_grace_period }} shutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }} {% if not kubelet_fail_swap_on %} diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index cbad203e493..f0290061d76 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -645,9 +645,9 @@ host_os: >- {{ ansible_system }} {%- endif -%} -# Sets the eventRecordQPS parameter in kubelet-config.yaml. The default value is 5 (see types.go) +# Sets the eventRecordQPS parameter in kubelet-config.yaml. # Setting it to 0 allows unlimited requests per second. -kubelet_event_record_qps: 5 +kubelet_event_record_qps: 50 proxy_env_defaults: http_proxy: "{{ http_proxy | default('') }}" From de2c4429a46bec5e73971ec57f13ad2034425d49 Mon Sep 17 00:00:00 2001 From: Alexander Date: Mon, 22 Jan 2024 18:00:34 +0100 Subject: [PATCH 044/103] =?UTF-8?q?Enable=20configuring=20mountOptions,=20?= =?UTF-8?q?reclaimPolicy=20and=20volumeBindingMode=20=E2=80=A6=20(#10450)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Enable configuring mountOptions, reclaimPolicy and volumeBindingMode for cinder-csi StorageClasses * Check if class.mount_options is defined at all, before generating the option list --- inventory/sample/group_vars/all/openstack.yml | 10 ++++++++++ .../templates/openstack-storage-class.yml.j2 | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/inventory/sample/group_vars/all/openstack.yml b/inventory/sample/group_vars/all/openstack.yml index 0fec79ad5b0..cac548f84c0 100644 --- a/inventory/sample/group_vars/all/openstack.yml +++ b/inventory/sample/group_vars/all/openstack.yml @@ -48,3 +48,13 @@ ## Make sure to source in the openstack credentials # cinder_csi_enabled: true # cinder_csi_controller_replicas: 1 +# storage_classes: +# - name: "cinder-csi" +# provisioner: "kubernetes.io/cinder" +# mount_options: +# - "discard" +# parameters: +# type: "thin" +# availability: "nova" +# reclaim_policy: "Delete" +# volume_binding_mode: "WaitForFirstConsumer" diff --git a/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 b/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 index 0551e153569..973353c4ce6 100644 --- a/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 +++ b/roles/kubernetes-apps/persistent_volumes/openstack/templates/openstack-storage-class.yml.j2 @@ -7,9 +7,21 @@ metadata: annotations: storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}" provisioner: kubernetes.io/cinder +{% if class.mount_options is defined %} +mountOptions: +{% for option in class.mount_options | default([]) %} + - "{{ option }}" +{% endfor %} +{% endif %} parameters: {% for key, value in (class.parameters | default({})).items() %} "{{ key }}": "{{ value }}" {% endfor %} +{% if class.reclaim_policy is defined %} +reclaimPolicy: "{{ class.reclaim_policy }}" +{% endif %} +{% if class.volume_binding_mode is defined %} +volumeBindingMode: "{{ class.volume_binding_mode }}" +{% endif %} allowVolumeExpansion: {{ expand_persistent_volumes }} {% endfor %} From 13e1f3389838e846ae8715ebf42531a53a329365 Mon Sep 17 00:00:00 2001 From: yun Date: Tue, 23 Jan 2024 01:00:52 +0800 Subject: [PATCH 045/103] Correct the POLY1305 cipher suites by adding the suffix _SHA256 (#10641) --- inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml | 4 ++-- roles/etcd/defaults/main.yml | 2 -- roles/kubernetes/control-plane/defaults/main/main.yml | 4 ++-- roles/kubernetes/node/defaults/main.yml | 4 ++-- tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml | 2 +- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 454ba303f22..118e1ff8fb1 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -340,7 +340,7 @@ persistent_volumes_enabled: false # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA # - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA @@ -348,7 +348,7 @@ persistent_volumes_enabled: false # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_RSA_WITH_RC4_128_SHA # - TLS_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 5eca0ee9e2b..814caed8ebb 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -107,9 +107,7 @@ etcd_retries: 4 # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 # - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 -# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 # - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # ETCD 3.5.x issue diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 19503817069..7c2171327b1 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -203,7 +203,7 @@ secrets_encryption_query: "resources[*].providers[0].{{ kube_encryption_algorith # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA # - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA @@ -211,7 +211,7 @@ secrets_encryption_query: "resources[*].providers[0].{{ kube_encryption_algorith # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_RSA_WITH_RC4_128_SHA # - TLS_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 0522187b95a..643551d9eae 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -223,7 +223,7 @@ azure_cloud: AzurePublicCloud # - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA # - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA @@ -231,7 +231,7 @@ azure_cloud: AzurePublicCloud # - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 # - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA # - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 -# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 +# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 # - TLS_ECDHE_RSA_WITH_RC4_128_SHA # - TLS_RSA_WITH_3DES_EDE_CBC_SHA # - TLS_RSA_WITH_AES_128_CBC_SHA diff --git a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml index e4bf63da025..d8dcc1f8e6d 100644 --- a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml +++ b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml @@ -29,7 +29,7 @@ tls_min_version: VersionTLS12 tls_cipher_suites: - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 # enable encryption at rest kube_encrypt_secret_data: true From 4fc1fc729e87fdbc2b27d06b742f52717336cab1 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 22 Jan 2024 18:00:59 +0100 Subject: [PATCH 046/103] Remove dead link in README for debian jessie (#10827) --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d69052da15e..2565680a713 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,6 @@ vagrant up - [Vagrant install](docs/vagrant.md) - [Flatcar Container Linux bootstrap](docs/flatcar.md) - [Fedora CoreOS bootstrap](docs/fcos.md) -- [Debian Jessie setup](docs/debian.md) - [openSUSE setup](docs/opensuse.md) - [Downloaded artifacts](docs/downloads.md) - [Cloud providers](docs/cloud.md) From b4dd8b4313f8519a1db4aa0db97fa302dc5f0062 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 22 Jan 2024 18:20:44 +0100 Subject: [PATCH 047/103] Hopefully final batches of ipaddr deprecation cleanup (#10822) --- inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml | 6 +++--- playbooks/ansible_version.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 118e1ff8fb1..68233b4c8cb 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -117,7 +117,7 @@ kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112 kube_network_node_prefix_ipv6: 120 # The port the API Server will be listening on. -kube_apiserver_ip: "{{ kube_service_addresses | ipaddr('net') | ipaddr(1) | ipaddr('address') }}" +kube_apiserver_ip: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" kube_apiserver_port: 6443 # (https) # Kube-proxy proxyMode configuration. @@ -219,8 +219,8 @@ resolvconf_mode: host_resolvconf # Deploy netchecker app to verify DNS resolve as an HTTP service deploy_netchecker: false # Ip address of the kubernetes skydns service -skydns_server: "{{ kube_service_addresses | ipaddr('net') | ipaddr(3) | ipaddr('address') }}" -skydns_server_secondary: "{{ kube_service_addresses | ipaddr('net') | ipaddr(4) | ipaddr('address') }}" +skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}" +skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}" dns_domain: "{{ cluster_name }}" ## Container runtime diff --git a/playbooks/ansible_version.yml b/playbooks/ansible_version.yml index f6cebdff73d..1fea52356ef 100644 --- a/playbooks/ansible_version.yml +++ b/playbooks/ansible_version.yml @@ -21,7 +21,7 @@ - name: "Check that python netaddr is installed" assert: msg: "Python netaddr is not present" - that: "'127.0.0.1' | ipaddr" + that: "'127.0.0.1' | ansible.utils.ipaddr" tags: - check From 89d42a7716cd7568f1a3749eada4e43685368e6a Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 22 Jan 2024 18:36:16 +0100 Subject: [PATCH 048/103] Fix coredns_dual usage (#10821) --- roles/kubernetes-apps/ansible/tasks/coredns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/ansible/tasks/coredns.yml b/roles/kubernetes-apps/ansible/tasks/coredns.yml index 8d5aad25531..897c6189ff5 100644 --- a/roles/kubernetes-apps/ansible/tasks/coredns.yml +++ b/roles/kubernetes-apps/ansible/tasks/coredns.yml @@ -36,7 +36,7 @@ - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment } - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc } - { name: dns-autoscaler, src: dns-autoscaler.yml, file: coredns-autoscaler-secondary.yml, type: deployment } - - { name: coredns, file: coredns-poddisruptionbudget.yml, type: poddisruptionbudget, condition: coredns_pod_disruption_budget } + - { name: coredns, src: coredns-poddisruptionbudget.yml, file: coredns-poddisruptionbudget-secondary.yml, type: poddisruptionbudget, condition: coredns_pod_disruption_budget } register: coredns_secondary_manifests vars: clusterIP: "{{ skydns_server_secondary }}" From a88bad7947022612a25847b1737c589c35ceb11b Mon Sep 17 00:00:00 2001 From: Louis Tu <92532497+tu1h@users.noreply.github.com> Date: Tue, 23 Jan 2024 14:42:33 +0800 Subject: [PATCH 049/103] Add scheduler plugins support (#10747) Signed-off-by: tu1h --- docs/kubernetes-apps/scheduler_plugins.md | 49 +++++ roles/kubernetes-apps/meta/main.yml | 8 + .../scheduler_plugins/defaults/main.yml | 29 +++ .../scheduler_plugins/tasks/main.yml | 68 ++++++ ...appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 | 197 ++++++++++++++++++ .../templates/cm-scheduler-plugins.yaml.j2 | 28 +++ .../deploy-scheduler-plugins.yaml.j2 | 74 +++++++ .../templates/namespace.yaml.j2 | 7 + ....diktyo.x-k8s.io_networktopologies.yaml.j2 | 148 +++++++++++++ .../templates/rbac-scheduler-plugins.yaml.j2 | 140 +++++++++++++ .../templates/sa-scheduler-plugins.yaml.j2 | 11 + .../scheduling.x-k8s.io_elasticquotas.yaml.j2 | 82 ++++++++ .../scheduling.x-k8s.io_podgroups.yaml.j2 | 97 +++++++++ ...node.k8s.io_noderesourcetopologies.yaml.j2 | 153 ++++++++++++++ .../defaults/main/download.yml | 12 ++ .../kubespray-defaults/defaults/main/main.yml | 3 + tests/files/packet_almalinux8-calico.yml | 3 + 17 files changed, 1109 insertions(+) create mode 100644 docs/kubernetes-apps/scheduler_plugins.md create mode 100644 roles/kubernetes-apps/scheduler_plugins/defaults/main.yml create mode 100644 roles/kubernetes-apps/scheduler_plugins/tasks/main.yml create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 create mode 100644 roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 diff --git a/docs/kubernetes-apps/scheduler_plugins.md b/docs/kubernetes-apps/scheduler_plugins.md new file mode 100644 index 00000000000..85e37910ad5 --- /dev/null +++ b/docs/kubernetes-apps/scheduler_plugins.md @@ -0,0 +1,49 @@ +# Scheduler plugins for Kubernetes + +[scheduler-plugins](https://github.com/kubernetes-sigs/scheduler-plugins) is out-of-tree scheduler plugins based on the [scheduler framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/). + +The kube-scheduler binary includes a list of plugins: + +- [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta] +- [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta] +- [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta] +- [NodeResouceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta] +- [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha] +- [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha] +- [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample] +- [CrossNodePreemption](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/crossnodepreemption/README.md) [Sample] +- [PodState](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/podstate/README.md) [Sample] +- [QualityOfService](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/qos/README.md) [Sample] + +Currently, we use [helm chart](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#installing-the-chart) to install the scheduler plugins, so that a second scheduler would be created and running. **Note that running multi-scheduler will inevitably encounter resource conflicts when the cluster is short of resources**. + +## Compatibility Matrix + +There are requirements for the version of Kubernetes, please see [Compatibility Matrix +](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master?tab=readme-ov-file#compatibility-matrix). It deserves our attention. + +| Scheduler Plugins | Compiled With K8s Version | +| ----------------- | ------------------------- | +| v0.27.8 | v1.27.8 | +| v0.26.8 | v1.26.7 | + +## Turning it on + + The `scheduler_plugins_enabled` option is used to enable the installation of scheduler plugins. + + You can enable or disable some plugins by setting the `scheduler_plugins_enabled_plugins` or `scheduler_plugins_disabled_plugins` option. They must be in the list we mentioned above. + + In addition, to use custom plugin configuration, set a value for `scheduler_plugins_plugin_config` option. + + For example, for Coscheduling plugin, you want to customize the permit waiting timeout to 10 seconds: + + ```yaml + scheduler_plugins_plugin_config: + - name: Coscheduling + args: + permitWaitingTimeSeconds: 10 # default is 60 + ``` + +## Leverage plugin + + Once the plugin is installed, we can apply CRs into cluster. For example, if using `CoScheduling`, we can apply the CR and test the deployment in the [example](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/doc/install.md#test-coscheduling). diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 9c19fdebd76..1b9cd6be8cf 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -124,3 +124,11 @@ dependencies: - inventory_hostname == groups['kube_control_plane'][0] tags: - argocd + + - role: kubernetes-apps/scheduler_plugins + when: + - scheduler_plugins_enabled + - kube_major_version is version('v1.28', '<') + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins diff --git a/roles/kubernetes-apps/scheduler_plugins/defaults/main.yml b/roles/kubernetes-apps/scheduler_plugins/defaults/main.yml new file mode 100644 index 00000000000..43f50f544a5 --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/defaults/main.yml @@ -0,0 +1,29 @@ +--- +scheduler_plugins_enabled: false + +scheduler_plugins_namespace: scheduler-plugins + +scheduler_plugins_controller_replicas: 1 + +scheduler_plugins_scheduler_replicas: 1 + +# The default is determined by the number of control plane nodes. +scheduler_plugins_scheduler_leader_elect: "{{ ((groups['kube_control_plane'] | length) > 1) }}" + +# Plugins to enable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info. +scheduler_plugins_enabled_plugins: + - Coscheduling + - CapacityScheduling + - NodeResourceTopologyMatch + - NodeResourcesAllocatable + +# Plugins to disable. See https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests/install/charts/as-a-second-scheduler/README.md#configuration for more info. +scheduler_plugins_disabled_plugins: + - PrioritySort + +# Customize the enabled plugins' config. +# Refer to the "pluginConfig" section of https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/manifests//scheduler-config.yaml. +scheduler_plugins_plugin_config: + - name: Coscheduling + args: + permitWaitingTimeSeconds: 10 # default is 60 diff --git a/roles/kubernetes-apps/scheduler_plugins/tasks/main.yml b/roles/kubernetes-apps/scheduler_plugins/tasks/main.yml new file mode 100644 index 00000000000..d17b19128ca --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/tasks/main.yml @@ -0,0 +1,68 @@ +--- +- name: Scheduler Plugins | Ensure dir exists + file: + path: "{{ kube_config_dir }}/scheduler-plugins" + state: directory + owner: root + group: root + mode: 0755 + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/scheduler-plugins/{{ item.file }}" + mode: 0644 + with_items: + - { name: appgroup, file: appgroup.diktyo.x-k8s.io_appgroups.yaml, type: crd } + - { name: networktopology, file: networktopology.diktyo.x-k8s.io_networktopologies.yaml, type: crd } + - { name: elasticquotas, file: scheduling.x-k8s.io_elasticquotas.yaml, type: crd } + - { name: podgroups, file: scheduling.x-k8s.io_podgroups.yaml, type: crd } + - { name: noderesourcetopologies, file: topology.node.k8s.io_noderesourcetopologies.yaml, type: crd } + - { name: namespace, file: namespace.yaml, type: namespace } + - { name: sa, file: sa-scheduler-plugins.yaml, type: serviceaccount } + - { name: rbac, file: rbac-scheduler-plugins.yaml, type: rbac } + - { name: cm, file: cm-scheduler-plugins.yaml, type: configmap } + - { name: deploy, file: deploy-scheduler-plugins.yaml, type: deployment } + register: scheduler_plugins_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/scheduler-plugins/{{ item.item.file }}" + state: "latest" + with_items: "{{ scheduler_plugins_manifests.results }}" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Wait for controller pods to be ready + command: "{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l app=scheduler-plugins-controller -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors + register: controller_pods_not_ready + until: controller_pods_not_ready.stdout.find("scheduler-plugins-controller")==-1 + retries: 30 + delay: 10 + ignore_errors: true + changed_when: false + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins + +- name: Scheduler Plugins | Wait for scheduler pods to be ready + command: "{{ kubectl }} -n {{ scheduler_plugins_namespace }} get pods -l component=scheduler -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" # noqa ignore-errors + register: scheduler_pods_not_ready + until: scheduler_pods_not_ready.stdout.find("scheduler-plugins-scheduler")==-1 + retries: 30 + delay: 10 + ignore_errors: true + changed_when: false + when: inventory_hostname == groups['kube_control_plane'][0] + tags: + - scheduler_plugins diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 new file mode 100644 index 00000000000..757a3b12d39 --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/appgroup.diktyo.x-k8s.io_appgroups.yaml.j2 @@ -0,0 +1,197 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: appgroups.appgroup.diktyo.x-k8s.io +spec: + group: appgroup.diktyo.x-k8s.io + names: + kind: AppGroup + listKind: AppGroupList + plural: appgroups + shortNames: + - ag + singular: appgroup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: AppGroup is a collection of Pods belonging to the same application. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppGroupSpec defines the number of Pods and which Pods belong + to the group. + properties: + numMembers: + description: NumMembers defines the number of Pods belonging to the + App Group + format: int32 + minimum: 1 + type: integer + topologySortingAlgorithm: + description: The preferred Topology Sorting Algorithm + type: string + workloads: + description: Workloads defines the workloads belonging to the group + items: + description: AppGroupWorkload represents the Workloads belonging + to the App Group. + properties: + dependencies: + description: Dependencies of the Workload. + items: + description: DependenciesInfo contains information about one + dependency. + properties: + maxNetworkCost: + description: Max Network Cost between workloads + format: int64 + maximum: 10000 + minimum: 0 + type: integer + minBandwidth: + anyOf: + - type: integer + - type: string + description: MinBandwidth between workloads + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + workload: + description: Workload reference Info. + properties: + apiVersion: + description: ApiVersion defines the versioned schema + of an object. + type: string + kind: + description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name represents the workload, info: + http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: Namespace of the workload + type: string + selector: + description: Selector defines how to find Pods related + to the Workload (key = workload). (e.g., workload=w1) + type: string + required: + - kind + - name + - selector + type: object + required: + - workload + type: object + type: array + workload: + description: Workload reference Info. + properties: + apiVersion: + description: ApiVersion defines the versioned schema of + an object. + type: string + kind: + description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: Namespace of the workload + type: string + selector: + description: Selector defines how to find Pods related to + the Workload (key = workload). (e.g., workload=w1) + type: string + required: + - kind + - name + - selector + type: object + required: + - workload + type: object + type: array + required: + - numMembers + - topologySortingAlgorithm + - workloads + type: object + status: + description: AppGroupStatus defines the observed use. + properties: + runningWorkloads: + description: The number of actively running workloads (e.g., number + of pods). + format: int32 + minimum: 0 + type: integer + scheduleStartTime: + description: ScheduleStartTime of the group + format: date-time + type: string + topologyCalculationTime: + description: TopologyCalculationTime of the group + format: date-time + type: string + topologyOrder: + description: Topology order for TopSort plugin (QueueSort) + items: + description: AppGroupTopologyInfo represents the calculated order + for a given Workload. + properties: + index: + description: Topology index. + format: int32 + type: integer + workload: + description: Workload reference Info. + properties: + apiVersion: + description: ApiVersion defines the versioned schema of + an object. + type: string + kind: + description: 'Kind of the workload, info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name represents the workload, info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: Namespace of the workload + type: string + selector: + description: Selector defines how to find Pods related to + the Workload (key = workload). (e.g., workload=w1) + type: string + required: + - kind + - name + - selector + type: object + type: object + type: array + type: object + type: object + served: true + storage: true \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 new file mode 100644 index 00000000000..7e022e8895b --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/cm-scheduler-plugins.yaml.j2 @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: scheduler-config + namespace: {{ scheduler_plugins_namespace }} +data: + scheduler-config.yaml: | + apiVersion: kubescheduler.config.k8s.io/v1 + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: {{ scheduler_plugins_scheduler_leader_elect | bool | lower }} + profiles: + # Compose all plugins in one profile + - schedulerName: scheduler-plugins-scheduler + plugins: + multiPoint: + enabled: +{% for enabeld_plugin in scheduler_plugins_enabled_plugins %} + - name: {{ enabeld_plugin }} +{% endfor %} + disabled: +{% for disabled_plugin in scheduler_plugins_disabled_plugins %} + - name: {{ disabled_plugin }} +{% endfor %} +{% if scheduler_plugins_plugin_config is defined and scheduler_plugins_plugin_config | length != 0 %} + pluginConfig: +{{ scheduler_plugins_plugin_config | to_nice_yaml(indent=2, width=256) | indent(6, true) }} +{% endif %} \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 new file mode 100644 index 00000000000..114698a9419 --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/deploy-scheduler-plugins.yaml.j2 @@ -0,0 +1,74 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} + labels: + app: scheduler-plugins-controller +spec: + replicas: {{ scheduler_plugins_controller_replicas }} + selector: + matchLabels: + app: scheduler-plugins-controller + template: + metadata: + labels: + app: scheduler-plugins-controller + spec: + serviceAccountName: scheduler-plugins-controller + containers: + - name: scheduler-plugins-controller + image: {{ scheduler_plugins_controller_image_repo }}:{{ scheduler_plugins_controller_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + component: scheduler + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +spec: + selector: + matchLabels: + component: scheduler + replicas: {{ scheduler_plugins_scheduler_replicas }} + template: + metadata: + labels: + component: scheduler + spec: + serviceAccountName: scheduler-plugins-scheduler + containers: + - command: + - /bin/kube-scheduler + - --config=/etc/kubernetes/scheduler-config.yaml + image: {{ scheduler_plugins_scheduler_image_repo }}:{{ scheduler_plugins_scheduler_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + livenessProbe: + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 15 + name: scheduler-plugins-scheduler + readinessProbe: + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + resources: + requests: + cpu: '0.1' + securityContext: + privileged: false + volumeMounts: + - name: scheduler-config + mountPath: /etc/kubernetes + readOnly: true + hostNetwork: false + hostPID: false + volumes: + - name: scheduler-config + configMap: + name: scheduler-config \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 new file mode 100644 index 00000000000..d54ae66fd80 --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/namespace.yaml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ scheduler_plugins_namespace }} + labels: + name: {{ scheduler_plugins_namespace }} \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 new file mode 100644 index 00000000000..e33157c0f2f --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/networktopology.diktyo.x-k8s.io_networktopologies.yaml.j2 @@ -0,0 +1,148 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/432 # edited manually + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: networktopologies.networktopology.diktyo.x-k8s.io +spec: + group: networktopology.diktyo.x-k8s.io + names: + kind: NetworkTopology + listKind: NetworkTopologyList + plural: networktopologies + shortNames: + - nt + singular: networktopology + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NetworkTopology defines network costs in the cluster between + regions and zones + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkTopologySpec defines the zones and regions of the + cluster. + properties: + configmapName: + description: ConfigmapName to be used for cost calculation + type: string + weights: + description: The manual defined weights of the cluster + items: + description: WeightInfo contains information about all network costs + for a given algorithm. + properties: + name: + description: Algorithm Name for network cost calculation (e.g., + userDefined) + type: string + topologyList: + description: TopologyList owns Costs between origins + items: + description: TopologyInfo contains information about network + costs for a particular Topology Key. + properties: + originList: + description: OriginList for a particular origin. + items: + description: OriginInfo contains information about network + costs for a particular Origin. + properties: + costList: + description: Costs for the particular origin. + items: + description: CostInfo contains information about + networkCosts. + properties: + bandwidthAllocated: + anyOf: + - type: integer + - type: string + description: Bandwidth allocated between origin + and destination. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + bandwidthCapacity: + anyOf: + - type: integer + - type: string + description: Bandwidth capacity between origin + and destination. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + destination: + description: Name of the destination (e.g., + Region Name, Zone Name). + type: string + networkCost: + description: Network Cost between origin and + destination (e.g., Dijkstra shortest path, + etc) + format: int64 + minimum: 0 + type: integer + required: + - destination + - networkCost + type: object + type: array + origin: + description: Name of the origin (e.g., Region Name, + Zone Name). + type: string + required: + - origin + type: object + type: array + topologyKey: + description: Topology key (e.g., "topology.kubernetes.io/region", + "topology.kubernetes.io/zone"). + type: string + required: + - originList + - topologyKey + type: object + type: array + required: + - name + - topologyList + type: object + type: array + required: + - configmapName + - weights + type: object + status: + description: NetworkTopologyStatus defines the observed use. + properties: + nodeCount: + description: The total number of nodes in the cluster + format: int64 + minimum: 0 + type: integer + weightCalculationTime: + description: The calculation time for the weights in the network topology + CRD + format: date-time + type: string + type: object + type: object + served: true + storage: true \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 new file mode 100644 index 00000000000..aa6f211d7de --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/rbac-scheduler-plugins.yaml.j2 @@ -0,0 +1,140 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: scheduler-plugins-scheduler +rules: +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch", "update"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] +- apiGroups: ["coordination.k8s.io"] + resourceNames: ["kube-scheduler"] + resources: ["leases"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["create"] +- apiGroups: [""] + resourceNames: ["kube-scheduler"] + resources: ["endpoints"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["delete", "get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["bindings", "pods/binding"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["patch", "update"] +- apiGroups: [""] + resources: ["replicationcontrollers", "services"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps", "extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumeclaims", "persistentvolumes"] + verbs: ["get", "list", "watch", "patch", "update"] +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +- apiGroups: ["storage.k8s.io"] + resources: ["csinodes", "storageclasses" , "csidrivers" , "csistoragecapacities"] + verbs: ["get", "list", "watch"] +- apiGroups: ["topology.node.k8s.io"] + resources: ["noderesourcetopologies"] + verbs: ["get", "list", "watch"] +# resources need to be updated with the scheduler plugins used +- apiGroups: ["scheduling.x-k8s.io"] + resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] +# for network-aware plugins add the following lines (scheduler-plugins v0.27.8) +#- apiGroups: [ "appgroup.diktyo.x-k8s.io" ] +# resources: [ "appgroups" ] +# verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ] +#- apiGroups: [ "networktopology.diktyo.x-k8s.io" ] +# resources: [ "networktopologies" ] +# verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scheduler-plugins-scheduler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: scheduler-plugins-scheduler +subjects: +- kind: ServiceAccount + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scheduler-plugins-controller +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch", "update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] +- apiGroups: ["topology.node.k8s.io"] + resources: ["noderesourcetopologies"] + verbs: ["get", "list", "watch"] +# resources need to be updated with the scheduler plugins used +- apiGroups: ["scheduling.x-k8s.io"] + resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scheduler-plugins-controller +subjects: +- kind: ServiceAccount + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} +roleRef: + kind: ClusterRole + name: scheduler-plugins-controller + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sched-plugins::extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +- kind: ServiceAccount + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 new file mode 100644 index 00000000000..6c25e18090c --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/sa-scheduler-plugins.yaml.j2 @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scheduler-plugins-scheduler + namespace: {{ scheduler_plugins_namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scheduler-plugins-controller + namespace: {{ scheduler_plugins_namespace }} \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 new file mode 100644 index 00000000000..d63f57209f4 --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_elasticquotas.yaml.j2 @@ -0,0 +1,82 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/52 + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: elasticquotas.scheduling.x-k8s.io +spec: + group: scheduling.x-k8s.io + names: + kind: ElasticQuota + listKind: ElasticQuotaList + plural: elasticquotas + shortNames: + - eq + - eqs + singular: elasticquota + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ElasticQuota sets elastic quota restrictions per namespace + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ElasticQuotaSpec defines the Min and Max for Quota. + properties: + max: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Max is the set of desired max limits for each named resource. + The usage of max is based on the resource configurations of successfully + scheduled pods. + type: object + min: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Min is the set of desired guaranteed limits for each + named resource. + type: object + type: object + status: + description: ElasticQuotaStatus defines the observed use. + properties: + used: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Used is the current observed total usage of the resource + in the namespace. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 new file mode 100644 index 00000000000..3767cf96291 --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/scheduling.x-k8s.io_podgroups.yaml.j2 @@ -0,0 +1,97 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/50 + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: podgroups.scheduling.x-k8s.io +spec: + group: scheduling.x-k8s.io + names: + kind: PodGroup + listKind: PodGroupList + plural: podgroups + shortNames: + - pg + - pgs + singular: podgroup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: PodGroup is a collection of Pod; used for batch workload. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of the pod group. + properties: + minMember: + description: MinMember defines the minimal number of members/tasks + to run the pod group; if there's not enough resources to start all + tasks, the scheduler will not start anyone. + format: int32 + type: integer + minResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: MinResources defines the minimal resource of members/tasks + to run the pod group; if there's not enough resources to start all + tasks, the scheduler will not start anyone. + type: object + scheduleTimeoutSeconds: + description: ScheduleTimeoutSeconds defines the maximal time of members/tasks + to wait before run the pod group; + format: int32 + type: integer + type: object + status: + description: Status represents the current information about a pod group. + This data may not be up to date. + properties: + failed: + description: The number of pods which reached phase Failed. + format: int32 + type: integer + occupiedBy: + description: OccupiedBy marks the workload (e.g., deployment, statefulset) + UID that occupy the podgroup. It is empty if not initialized. + type: string + phase: + description: Current phase of PodGroup. + type: string + running: + description: The number of actively running pods. + format: int32 + type: integer + scheduleStartTime: + description: ScheduleStartTime of the group + format: date-time + type: string + succeeded: + description: The number of pods which reached phase Succeeded. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 b/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 new file mode 100644 index 00000000000..d83ef0b9b5c --- /dev/null +++ b/roles/kubernetes-apps/scheduler_plugins/templates/topology.node.k8s.io_noderesourcetopologies.yaml.j2 @@ -0,0 +1,153 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870 + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: noderesourcetopologies.topology.node.k8s.io +spec: + group: topology.node.k8s.io + names: + kind: NodeResourceTopology + listKind: NodeResourceTopologyList + plural: noderesourcetopologies + shortNames: + - node-res-topo + singular: noderesourcetopology + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: NodeResourceTopology describes node resources and their topology. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + topologyPolicies: + description: 'DEPRECATED (to be removed in v1beta1): use top level attributes + if needed' + items: + type: string + type: array + zones: + description: ZoneList contains an array of Zone objects. + items: + description: Zone represents a resource topology zone, e.g. socket, + node, die or core. + properties: + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + costs: + description: CostList contains an array of CostInfo objects. + items: + description: CostInfo describes the cost (or distance) between + two Zones. + properties: + name: + type: string + value: + format: int64 + type: integer + required: + - name + - value + type: object + type: array + name: + type: string + parent: + type: string + resources: + description: ResourceInfoList contains an array of ResourceInfo + objects. + items: + description: ResourceInfo contains information about one resource + type. + properties: + allocatable: + anyOf: + - type: integer + - type: string + description: Allocatable quantity of the resource, corresponding + to allocatable in node status, i.e. total amount of this + resource available to be used by pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + available: + anyOf: + - type: integer + - type: string + description: Available is the amount of this resource currently + available for new (to be scheduled) pods, i.e. Allocatable + minus the resources reserved by currently running pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + capacity: + anyOf: + - type: integer + - type: string + description: Capacity of the resource, corresponding to capacity + in node status, i.e. total amount of this resource that + the node has. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: Name of the resource. + type: string + required: + - allocatable + - available + - capacity + - name + type: object + type: array + type: + type: string + required: + - name + - type + type: object + type: array + required: + - zones + type: object + served: true + storage: true \ No newline at end of file diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 676989ea310..cc8d676f75c 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -156,6 +156,13 @@ crio_supported_versions: v1.27: v1.27.1 crio_version: "{{ crio_supported_versions[kube_major_version] }}" +# Scheduler plugins doesn't build for K8s 1.28 yet +scheduler_plugins_supported_versions: + v1.28: 0 + v1.27: v0.27.8 + v1.26: v0.26.7 +scheduler_plugins_version: "{{ scheduler_plugins_supported_versions[kube_major_version] }}" + yq_version: "v4.35.2" # Download URLs @@ -294,6 +301,11 @@ dnsautoscaler_version: v1.8.8 dnsautoscaler_image_repo: "{{ kube_image_repo }}/cpa/cluster-proportional-autoscaler" dnsautoscaler_image_tag: "{{ dnsautoscaler_version }}" +scheduler_plugins_controller_image_repo: "{{ kube_image_repo }}/scheduler-plugins/controller" +scheduler_plugins_controller_image_tag: "{{ scheduler_plugins_version }}" +scheduler_plugins_scheduler_image_repo: "{{ kube_image_repo }}/scheduler-plugins/kube-scheduler" +scheduler_plugins_scheduler_image_tag: "{{ scheduler_plugins_version }}" + registry_version: "2.8.1" registry_image_repo: "{{ docker_image_repo }}/library/registry" registry_image_tag: "{{ registry_version }}" diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index f0290061d76..f7eb5b37345 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -681,3 +681,6 @@ sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" system_upgrade: false system_upgrade_reboot: on-upgrade # never, always + +# Enables or disables the scheduler plugins. +scheduler_plugins_enabled: false diff --git a/tests/files/packet_almalinux8-calico.yml b/tests/files/packet_almalinux8-calico.yml index 63cf8bf640e..648a72d1309 100644 --- a/tests/files/packet_almalinux8-calico.yml +++ b/tests/files/packet_almalinux8-calico.yml @@ -17,3 +17,6 @@ ntp_timezone: Etc/UTC ntp_manage_config: true ntp_tinker_panic: true ntp_force_sync_immediately: true + +# Scheduler plugins +scheduler_plugins_enabled: true From 2eb588bed943eebd95dd993266f0ffb0494719fb Mon Sep 17 00:00:00 2001 From: Daniel Strufe <2900921+dabeck@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:28:00 +0100 Subject: [PATCH 050/103] Update external huawei cloud controller to 0.26.6 (#10824) * Update huaweicloud controller to 0.26.6 See * Update huaweicloud sample to use 0.26.6 --- inventory/sample/group_vars/all/huaweicloud.yml | 2 +- .../external_cloud_controller/huaweicloud/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/sample/group_vars/all/huaweicloud.yml b/inventory/sample/group_vars/all/huaweicloud.yml index 20c72022771..b85e7c2acd8 100644 --- a/inventory/sample/group_vars/all/huaweicloud.yml +++ b/inventory/sample/group_vars/all/huaweicloud.yml @@ -14,4 +14,4 @@ ## The repo and tag of the external Huawei Cloud Controller image # external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com" -# external_huawei_cloud_controller_image_tag: "v0.26.3" +# external_huawei_cloud_controller_image_tag: "v0.26.6" diff --git a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml index 6d89c572677..f81bf1e2ec9 100644 --- a/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml +++ b/roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml @@ -16,4 +16,4 @@ external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}" ## arg2: "value2" external_huawei_cloud_controller_extra_args: {} external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com" -external_huawei_cloud_controller_image_tag: "v0.26.3" +external_huawei_cloud_controller_image_tag: "v0.26.6" From ab0163a3adce4feacb39d8b1619de478a67d9100 Mon Sep 17 00:00:00 2001 From: Maxime Leroy <19607336+maxime1907@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:46:13 +0100 Subject: [PATCH 051/103] fix(kubernetes): taint nodes with kubectl (#10705) Signed-off-by: Maxime Leroy <19607336+maxime1907@users.noreply.github.com> --- docs/vars.md | 2 +- playbooks/cluster.yml | 1 + playbooks/scale.yml | 1 + playbooks/upgrade_cluster.yml | 2 ++ roles/kubernetes/node-taint/tasks/main.yml | 35 +++++++++++++++++++ .../node/templates/kubelet.env.v1beta1.j2 | 12 +------ 6 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 roles/kubernetes/node-taint/tasks/main.yml diff --git a/docs/vars.md b/docs/vars.md index 959260e3138..9c9f003998d 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -245,7 +245,7 @@ node_labels: label2_name: label2_value ``` -* *node_taints* - Taints applied to nodes via kubelet --register-with-taints parameter. +* *node_taints* - Taints applied to nodes via `kubectl taint node`. For example, taints can be set in the inventory as variables or more widely in group_vars. *node_taints* has to be defined as a list of strings in format `key=value:effect`, e.g.: diff --git a/playbooks/cluster.yml b/playbooks/cluster.yml index a6fd770b957..c433a8c6928 100644 --- a/playbooks/cluster.yml +++ b/playbooks/cluster.yml @@ -48,6 +48,7 @@ - { role: kubespray-defaults } - { role: kubernetes/kubeadm, tags: kubeadm} - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } - { role: network_plugin, tags: network } - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver } diff --git a/playbooks/scale.yml b/playbooks/scale.yml index b8f87f484e8..171e378328d 100644 --- a/playbooks/scale.yml +++ b/playbooks/scale.yml @@ -91,6 +91,7 @@ - { role: kubespray-defaults } - { role: kubernetes/kubeadm, tags: kubeadm } - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } - { role: network_plugin, tags: network } - name: Apply resolv.conf changes now that cluster DNS is up diff --git a/playbooks/upgrade_cluster.yml b/playbooks/upgrade_cluster.yml index a79cf0aa763..3180fec9310 100644 --- a/playbooks/upgrade_cluster.yml +++ b/playbooks/upgrade_cluster.yml @@ -55,6 +55,7 @@ - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true } - { role: kubernetes/client, tags: client } - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } - { role: kubernetes-apps/cluster_roles, tags: cluster-roles } - { role: kubernetes-apps, tags: csi-driver } - { role: upgrade/post-upgrade, tags: post-upgrade } @@ -87,6 +88,7 @@ - { role: kubernetes/node, tags: node } - { role: kubernetes/kubeadm, tags: kubeadm } - { role: kubernetes/node-label, tags: node-label } + - { role: kubernetes/node-taint, tags: node-taint } - { role: upgrade/post-upgrade, tags: post-upgrade } - name: Patch Kubernetes for Windows diff --git a/roles/kubernetes/node-taint/tasks/main.yml b/roles/kubernetes/node-taint/tasks/main.yml new file mode 100644 index 00000000000..0766dc4396c --- /dev/null +++ b/roles/kubernetes/node-taint/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: Set role and inventory node taint to empty list + set_fact: + role_node_taints: [] + inventory_node_taints: [] + +- name: Node taint for nvidia GPU nodes + set_fact: + role_node_taints: "{{ role_node_taints + ['nvidia.com/gpu=:NoSchedule'] }}" + when: + - nvidia_gpu_nodes is defined + - nvidia_accelerator_enabled | bool + - inventory_hostname in nvidia_gpu_nodes + +- name: Populate inventory node taint + set_fact: + inventory_node_taints: "{{ inventory_node_taints + ['%s' | format(item)] }}" + loop: "{{ node_taints | d([]) }}" + when: + - node_taints is defined + - node_taints is not string + - node_taints is not mapping + - node_taints is iterable +- debug: # noqa name[missing] + var: role_node_taints +- debug: # noqa name[missing] + var: inventory_node_taints + +- name: Set taint to node + command: >- + {{ kubectl }} taint node {{ kube_override_hostname | default(inventory_hostname) }} {{ (role_node_taints + inventory_node_taints) | join(' ') }} --overwrite=true + delegate_to: "{{ groups['kube_control_plane'][0] }}" + changed_when: false + when: + - (role_node_taints + inventory_node_taints) | length > 0 diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index b8a22fd1ecb..a5aa369df57 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -15,17 +15,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --runtime-cgroups={{ kubelet_runtime_cgroups }} \ {% endset %} -{# Kubelet node taints for gpu #} -{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} -{% if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %} -{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} -{% elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %} -{% set node_taints = [] %} -{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} -{% endif %} -{% endif %} - -KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" {% if kubelet_flexvolumes_plugins_dir is defined %} KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}" {% endif %} From 0e26f6f3e288e0e251ca070866bc239cc69d8799 Mon Sep 17 00:00:00 2001 From: Simon Wessel <9195792+simon-wessel@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:41:20 +0100 Subject: [PATCH 052/103] chore: improve performance of python script for hash download (#10335) The old version of the script downloaded all binaries and generated file checksums locally. This was a slow process since all binaries of all architectures needed to be downloaded. The new version simply downloads the .sha256 files containing the binary checksum in text form which saves a lot of traffic and time. --- scripts/download_hash.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/scripts/download_hash.py b/scripts/download_hash.py index 86365e33323..a136e2065ea 100644 --- a/scripts/download_hash.py +++ b/scripts/download_hash.py @@ -4,22 +4,21 @@ # run this script to update roles/kubespray-defaults/defaults/main/download.yml # with new hashes. -import hashlib import sys import requests from ruamel.yaml import YAML -MAIN_YML = "../roles/kubespray-defaults/defaults/main/download.yml" +CHECKSUMS_YML = "../roles/kubespray-defaults/defaults/main/checksums.yml" -def open_main_yaml(): +def open_checksums_yaml(): yaml = YAML() yaml.explicit_start = True yaml.preserve_quotes = True yaml.width = 4096 - with open(MAIN_YML, "r") as main_yml: - data = yaml.load(main_yml) + with open(CHECKSUMS_YML, "r") as checksums_yml: + data = yaml.load(checksums_yml) return data, yaml @@ -28,7 +27,7 @@ def download_hash(versions): architectures = ["arm", "arm64", "amd64", "ppc64le"] downloads = ["kubelet", "kubectl", "kubeadm"] - data, yaml = open_main_yaml() + data, yaml = open_checksums_yaml() for download in downloads: checksum_name = f"{download}_checksums" @@ -36,15 +35,25 @@ def download_hash(versions): for version in versions: if not version.startswith("v"): version = f"v{version}" - url = f"https://dl.k8s.io/release/{version}/bin/linux/{arch}/{download}" - download_file = requests.get(url, allow_redirects=True) - download_file.raise_for_status() - sha256sum = hashlib.sha256(download_file.content).hexdigest() + url = f"https://dl.k8s.io/release/{version}/bin/linux/{arch}/{download}.sha256" + hash_file = requests.get(url, allow_redirects=True) + if hash_file.status_code == 404: + print(f"Unable to find hash file for release {version} (arch: {arch})") + continue + if hash_file.status_code != 200: + raise Exception(f"Received a non-200 HTTP response code: {hash_file.status_code} (arch: {arch}, version: {version})") + sha256sum = hash_file.content.decode().strip() + if len(sha256sum) != 64: + raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (arch: {arch}, version: {version})") + if checksum_name not in data: + data[checksum_name] = {} + if arch not in data[checksum_name]: + data[checksum_name][arch] = {} data[checksum_name][arch][version] = sha256sum - with open(MAIN_YML, "w") as main_yml: - yaml.dump(data, main_yml) - print(f"\n\nUpdated {MAIN_YML}\n") + with open(CHECKSUMS_YML, "w") as checksums_yml: + yaml.dump(data, checksums_yml) + print(f"\n\nUpdated {CHECKSUMS_YML}\n") def usage(): From c80f2cd573b41b7cef54b621beb800b95ad8f98a Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 24 Jan 2024 06:31:22 +0100 Subject: [PATCH 053/103] Allow the DNS stack to be backward compatible with an old dns_domain (#10630) Handle all old dns domains: - for nodelocaldns: in the same server block as the current dns_domain - for coredns: uffix rewrite of each of the old dns domains to the current one --- docs/dns-stack.md | 16 ++++++++++++++++ roles/kubernetes-apps/ansible/defaults/main.yml | 4 ++++ .../ansible/templates/coredns-config.yml.j2 | 3 +++ .../ansible/templates/nodelocaldns-config.yml.j2 | 2 +- 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 3ba3669ff19..ae560d4a8c1 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -143,6 +143,22 @@ coredns_default_zone_cache_block: | } ``` +### Handle old/extra dns_domains + +If you need to change the dns_domain of your cluster for whatever reason (switching to or from `cluster.local` for example), +and you have workloads that embed it in their configuration you can use the variable `old_dns_domains`. +This will add some configuration to coredns and nodelocaldns to ensure the DNS requests using the old domain are handled correctly. +Example: + +```yaml +old_dns_domains: +- example1.com +- example2.com +dns_domain: cluster.local +``` + +will make `my-svc.my-ns.svc.example1.com`, `my-svc.my-ns.svc.example2.com` and `my-svc.my-ns.svc.cluster.local` have the same DNS answer. + ### systemd_resolved_disable_stub_listener Whether or not to set `DNSStubListener=no` when using systemd-resolved. Defaults to `true` on Flatcar. diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 0050ce05b75..cb2317b1e05 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -37,6 +37,10 @@ coredns_pod_disruption_budget_max_unavailable: "30%" # coredns_additional_error_config: | # consolidate 5m ".* i/o timeout$" warning +# Configure coredns and nodelocaldns to correctly answer DNS queries when you changed +# your 'dns_domain' and some workloads used it directly. +old_dns_domains: [] + # dns_upstream_forward_extra_opts apply to coredns forward section as well as nodelocaldns upstream target forward section # dns_upstream_forward_extra_opts: # policy: sequential diff --git a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 index acab26628c8..587a4e65d2a 100644 --- a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 @@ -49,6 +49,9 @@ data: {% if coredns_rewrite_block is defined %} {{ coredns_rewrite_block | indent(width=8, first=False) }} {% endif %} +{% for old_dns_domain in old_dns_domains %} + rewrite name suffix {{ old_dns_domain }} {{ dns_domain }} answer auto +{% endfor %} ready kubernetes {{ dns_domain }} {% if coredns_kubernetes_extra_domains is defined %}{{ coredns_kubernetes_extra_domains }} {% endif %}{% if enable_coredns_reverse_dns_lookups %}in-addr.arpa ip6.arpa {% endif %}{ pods insecure diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 index b15ea89e996..e8ce54529c0 100644 --- a/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 @@ -32,7 +32,7 @@ data: } {% endfor %} {% endif %} - {{ dns_domain }}:53 { + {{ ([dns_domain] + old_dns_domains) | join(' ') }}:53 { errors cache { success 9984 30 From 758d34a7d15e8aefcd96da36fe3fc83bb3eb0525 Mon Sep 17 00:00:00 2001 From: kimsehwan96 Date: Wed, 24 Jan 2024 15:38:35 +0900 Subject: [PATCH 054/103] Fix typo mistake in roles/kubernetes/control-plane/tasks/define-first-kube-control.yml - Fix 'Set fact joined_control_panes' into 'Set fact joined_control_planes' --- .../control-plane/tasks/define-first-kube-control.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml b/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml index 64e2de785c7..36996da2e89 100644 --- a/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml +++ b/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml @@ -6,7 +6,7 @@ ignore_errors: yes changed_when: false -- name: Set fact joined_control_panes +- name: Set fact joined_control_planes set_fact: joined_control_planes: "{{ ((kube_control_planes_raw.stdout | from_json)['items']) | default([]) | map(attribute='metadata') | map(attribute='name') | list }}" delegate_to: item From 7863fde552dc7821d96914d88c209aa351b68ccc Mon Sep 17 00:00:00 2001 From: Ugur Can Ozturk <57688057+ugur99@users.noreply.github.com> Date: Thu, 25 Jan 2024 10:24:35 +0100 Subject: [PATCH 055/103] [apiserver-kubelet/tracing]: add distributed tracing config variables (#10795) * [apiserver-kubelet/tracing]: add distributed tracing config flags Signed-off-by: Ugur Ozturk * [apiserver-kubelet/tracing]: add distributed tracing config flags - fix Signed-off-by: Ugur Ozturk * [apiserver-kubelet/tracing]: add distributed tracing config flags - fix Signed-off-by: Ugur Ozturk --------- Signed-off-by: Ugur Ozturk --- .../control-plane/defaults/main/main.yml | 5 +++++ .../control-plane/tasks/kubeadm-setup.yml | 14 ++++++++++++++ .../templates/apiserver-tracing.yaml.j2 | 4 ++++ .../templates/kubeadm-config.v1beta3.yaml.j2 | 10 ++++++++++ roles/kubernetes/node/defaults/main.yml | 6 ++++++ .../node/templates/kubelet-config.v1beta1.yaml.j2 | 5 +++++ 6 files changed, 44 insertions(+) create mode 100644 roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 7c2171327b1..fd7047767cd 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -235,3 +235,8 @@ kubeadm_upgrade_auto_cert_renewal: true # Bash alias of kubectl to interact with Kubernetes cluster much easier # kubectl_alias: k + +## Enable distributed tracing for kube-apiserver +kube_apiserver_tracing: false +kube_apiserver_tracing_endpoint: 0.0.0.0:4317 +kube_apiserver_tracing_sampling_rate_per_million: 100 diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index dcad832ba89..1f4ff20a3c6 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -68,6 +68,20 @@ mode: 0640 when: kubernetes_audit_webhook | default(false) +- name: Create apiserver tracing config directory + file: + path: "{{ kube_config_dir }}/tracing" + state: directory + mode: 0640 + when: kube_apiserver_tracing + +- name: Write apiserver tracing config yaml + template: + src: apiserver-tracing.yaml.j2 + dest: "{{ kube_config_dir }}/tracing/apiserver-tracing.yaml" + mode: 0640 + when: kube_apiserver_tracing + # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. - name: Set kubeadm_config_api_fqdn define set_fact: diff --git a/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 new file mode 100644 index 00000000000..98decde86b8 --- /dev/null +++ b/roles/kubernetes/control-plane/templates/apiserver-tracing.yaml.j2 @@ -0,0 +1,4 @@ +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: TracingConfiguration +endpoint: {{ kube_apiserver_tracing_endpoint }} +samplingRatePerMillion: {{ kube_apiserver_tracing_sampling_rate_per_million }} \ No newline at end of file diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index cbb22182315..b11fb33431e 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -227,6 +227,9 @@ apiServer: {% if kubelet_rotate_server_certificates %} kubelet-certificate-authority: {{ kube_cert_dir }}/ca.crt {% endif %} +{% if kube_apiserver_tracing %} + tracing-config-file: {{ kube_config_dir }}/tracing/apiserver-tracing.yaml +{% endif %} {% if kubernetes_audit or kube_token_auth | default(true) or kube_webhook_token_auth | default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] ) or apiserver_extra_volumes or ssl_ca_dirs | length %} extraVolumes: {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce"] %} @@ -267,6 +270,13 @@ apiServer: readOnly: false pathType: DirectoryOrCreate {% endif %} +{% if kube_apiserver_tracing %} + - name: tracing + hostPath: {{ kube_config_dir }}/tracing + mountPath: {{ kube_config_dir }}/tracing + readOnly: true + pathType: DirectoryOrCreate +{% endif %} {% for volume in apiserver_extra_volumes %} - name: {{ volume.name }} hostPath: {{ volume.hostPath }} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 643551d9eae..b3f8fbf10fa 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -253,3 +253,9 @@ kube_proxy_ipvs_modules: conntrack_modules: - nf_conntrack - nf_conntrack_ipv4 + + +## Enable distributed tracing for kubelet +kubelet_tracing: false +kubelet_tracing_endpoint: 0.0.0.0:4317 +kubelet_tracing_sampling_rate_per_million: 100 \ No newline at end of file diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index 1cd00992af7..ba90fc9c8d3 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -166,3 +166,8 @@ topologyManagerPolicy: {{ kubelet_topology_manager_policy }} {% if kubelet_topology_manager_scope is defined %} topologyManagerScope: {{ kubelet_topology_manager_scope }} {% endif %} +{% if kubelet_tracing %} +tracing: + endpoint: {{ kubelet_tracing_endpoint }} + samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }} +{% endif %} \ No newline at end of file From a487667b9dc221e7f8ce42f2ea7166809fcfa93d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 25 Jan 2024 15:55:33 +0200 Subject: [PATCH 056/103] Make large-deployments.md link to downloads.md (#10840) --- docs/large-deployments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/large-deployments.md b/docs/large-deployments.md index 7acbff3d15e..4666d073eb6 100644 --- a/docs/large-deployments.md +++ b/docs/large-deployments.md @@ -9,7 +9,7 @@ For a large scaled deployments, consider the following configuration changes: * Override containers' `foo_image_repo` vars to point to intranet registry. * Override the ``download_run_once: true`` and/or ``download_localhost: true``. - See download modes for details. + See [Downloading binaries and containers](/docs/downloads.md) for details. * Adjust the `retry_stagger` global var as appropriate. It should provide sane load on a delegate (the first K8s control plane node) then retrying failed From 0ddf8721638beeca7539869f67e325682777f8c7 Mon Sep 17 00:00:00 2001 From: jandres - moscardo Date: Mon, 29 Jan 2024 06:07:17 +0100 Subject: [PATCH 057/103] Update upgrades.md with serial=1 for rolling updates (#10837) * Update upgrades.md modify env serial to have real rolling upgrades * Update upgrades.md change section for serial * Update docs/upgrades.md Co-authored-by: Kundan Kumar --------- Co-authored-by: Kundan Kumar --- docs/upgrades.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/upgrades.md b/docs/upgrades.md index 52dccba4283..97867fc10d5 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -59,6 +59,12 @@ Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7", GitCom Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-13T13:15:20Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"} ``` +You can control how many nodes are upgraded at the same time by modifying the ansible variable named `serial`, as explained [here](https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_strategies.html#setting-the-batch-size-with-serial). If you don't set this variable, it will upgrade the cluster nodes in batches of 20% of the available nodes. Setting `serial=1` would mean upgrade one node at a time. + +```ShellSession +ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=v1.20.7 -e "serial=1" +``` + ### Pausing the upgrade If you want to manually control the upgrade procedure, you can set some variables to pause the upgrade playbook. Pausing *before* upgrading each upgrade may be useful for inspecting pods running on that node, or performing manual actions on the node: From 4ea1a0132e567726a9b1ffcda1cf6b1a72ad118b Mon Sep 17 00:00:00 2001 From: Kundan Kumar Date: Mon, 29 Jan 2024 14:16:35 +0900 Subject: [PATCH 058/103] Updated vagrant.md (#10836) --- docs/vagrant.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/vagrant.md b/docs/vagrant.md index b7f702ce86d..e331c51129f 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -1,6 +1,6 @@ # Vagrant -Assuming you have Vagrant 2.0+ installed with virtualbox, libvirt/qemu or +Assuming you have Vagrant 2.0+ installed with virtualbox (it may work with libvirt/qemu or vmware, but is untested) you should be able to launch a 3 node Kubernetes cluster by simply running `vagrant up`. From 1d119f1a3c90e9af0d2d90c34c42f6f2545b71bd Mon Sep 17 00:00:00 2001 From: Saber <52651225+sabershahhoseini@users.noreply.github.com> Date: Tue, 30 Jan 2024 05:16:58 +0330 Subject: [PATCH 059/103] Fixed grammar (#10853) --- roles/kubernetes/preinstall/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index c18209e7ceb..eb33ed3db76 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -19,7 +19,7 @@ common_required_pkgs: - tar # Set to true if your network does not support IPv6 -# This maybe necessary for pulling Docker images from +# This may be necessary for pulling Docker images from # GCE docker repository disable_ipv6_dns: false @@ -105,7 +105,7 @@ ntp_driftfile: /var/lib/ntp/ntp.drift # Only takes effect when ntp_manage_config is true. ntp_tinker_panic: false -# Force sync time immediately after the ntp installed, which is useful in in newly installed system. +# Force sync time immediately after the ntp installed, which is useful in a newly installed system. ntp_force_sync_immediately: false # Set the timezone for your server. eg: "Etc/UTC","Etc/GMT-8". If not set, the timezone will not change. From ee8b909a678315e5b7848b962655d87a507af90b Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 30 Jan 2024 03:06:10 +0100 Subject: [PATCH 060/103] Download hash script: auto discover versions (#10849) * Download patches version automatically from a minor * Automate versions discovery for hash download * Small refactoring --- scripts/download_hash.py | 52 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/scripts/download_hash.py b/scripts/download_hash.py index a136e2065ea..61025c8bf5e 100644 --- a/scripts/download_hash.py +++ b/scripts/download_hash.py @@ -6,8 +6,11 @@ import sys +from itertools import count +from collections import defaultdict import requests from ruamel.yaml import YAML +from packaging.version import Version CHECKSUMS_YML = "../roles/kubespray-defaults/defaults/main/checksums.yml" @@ -23,33 +26,37 @@ def open_checksums_yaml(): return data, yaml -def download_hash(versions): +def download_hash(minors): architectures = ["arm", "arm64", "amd64", "ppc64le"] downloads = ["kubelet", "kubectl", "kubeadm"] data, yaml = open_checksums_yaml() + if not minors: + minors = {'.'.join(minor.split('.')[:-1]) for minor in data["kubelet_checksums"]["amd64"].keys()} for download in downloads: checksum_name = f"{download}_checksums" + data[checksum_name] = defaultdict(dict, data[checksum_name]) for arch in architectures: - for version in versions: - if not version.startswith("v"): - version = f"v{version}" - url = f"https://dl.k8s.io/release/{version}/bin/linux/{arch}/{download}.sha256" - hash_file = requests.get(url, allow_redirects=True) - if hash_file.status_code == 404: - print(f"Unable to find hash file for release {version} (arch: {arch})") - continue - if hash_file.status_code != 200: - raise Exception(f"Received a non-200 HTTP response code: {hash_file.status_code} (arch: {arch}, version: {version})") - sha256sum = hash_file.content.decode().strip() - if len(sha256sum) != 64: - raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (arch: {arch}, version: {version})") - if checksum_name not in data: - data[checksum_name] = {} - if arch not in data[checksum_name]: - data[checksum_name][arch] = {} - data[checksum_name][arch][version] = sha256sum + for minor in minors: + if not minor.startswith("v"): + minor = f"v{minor}" + for release in (f"{minor}.{patch}" for patch in count(start=0, step=1)): + if release in data[checksum_name][arch]: + continue + hash_file = requests.get(f"https://dl.k8s.io/release/{release}/bin/linux/{arch}/{download}.sha256", allow_redirects=True) + if hash_file.status_code == 404: + print(f"Unable to find {download} hash file for release {release} (arch: {arch})") + break + hash_file.raise_for_status() + sha256sum = hash_file.content.decode().strip() + if len(sha256sum) != 64: + raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (binary: {download}, arch: {arch}, release: 1.{minor}.{patch})") + data[checksum_name][arch][release] = sha256sum + data[checksum_name] = {arch : {r : releases[r] for r in sorted(releases.keys(), + key=lambda v : Version(v[1:]), + reverse=True)} + for arch, releases in data[checksum_name].items()} with open(CHECKSUMS_YML, "w") as checksums_yml: yaml.dump(data, checksums_yml) @@ -61,12 +68,7 @@ def usage(): def main(argv=None): - if not argv: - argv = sys.argv[1:] - if not argv: - usage() - return 1 - download_hash(argv) + download_hash(sys.argv[1:]) return 0 From e3ea19307a4290fb84ee1b3adff1004b8f714496 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 30 Jan 2024 07:31:40 +0100 Subject: [PATCH 061/103] Doc clarification: skipping patches releases is OK (#10850) --- docs/upgrades.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/upgrades.md b/docs/upgrades.md index 97867fc10d5..6c915c76585 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -103,9 +103,26 @@ ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_ve ## Multiple upgrades > **Warning** -> [Do not skip releases when upgrading--upgrade by one tag at a time.](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515) +> [Do not skip minor releases (patches releases are ok) when upgrading--upgrade by one tag at a +> time.](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515) -For instance, if you're on v2.6.0, then check out v2.7.0, run the upgrade, check out the next tag, and run the next upgrade, etc. +For instances, given the tag list: + +```console +$ git tag +v2.20.0 +v2.21.0 +v2.22.0 +v2.22.1 +v2.23.0 +v2.23.1 +v2.23.2 +v2.24.0 +... +``` + +v2.22.0 -> v2.23.2 -> v2.24.0 : ✓ +v.22.0 -> v2.24.0 : ✕ Assuming you don't explicitly define a kubernetes version in your k8s_cluster.yml, you simply check out the next tag and run the upgrade-cluster.yml playbook From 785366c2de50851b03749fc0eb5f33e3c4d63fe9 Mon Sep 17 00:00:00 2001 From: Takuya Murakami Date: Wed, 31 Jan 2024 17:57:23 +0900 Subject: [PATCH 062/103] [kubernetes] Support kubernetes 1.29 (#10820) * [kubernetes] Make kubernetes 1.29.1 default * [cri-o]: support cri-o 1.29 Use "crio status" instead of "crio-status" for cri-o >=1.29.0 * Remove GAed feature gates SecCompDefault The SecCompDefault feature gate was removed since k8s 1.29 https://github.com/kubernetes/kubernetes/pull/121246 --- README.md | 4 ++-- docs/hardening.md | 2 +- inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml | 2 +- roles/container-engine/cri-o/defaults/main.yml | 7 ------- roles/container-engine/cri-o/tasks/load_vars.yml | 8 ++++++++ roles/container-engine/cri-o/tasks/main.yaml | 5 ++++- roles/container-engine/cri-o/tasks/reset.yml | 3 +++ roles/container-engine/cri-o/vars/v1.28.yml | 9 +++++++++ roles/container-engine/cri-o/vars/v1.29.yml | 9 +++++++++ roles/kubernetes/node/tasks/facts.yml | 2 +- roles/kubespray-defaults/defaults/main/download.yml | 4 ++-- roles/kubespray-defaults/defaults/main/main.yml | 2 +- .../packet_ubuntu20-calico-all-in-one-hardening.yml | 2 +- 13 files changed, 42 insertions(+), 17 deletions(-) create mode 100644 roles/container-engine/cri-o/tasks/load_vars.yml create mode 100644 roles/container-engine/cri-o/vars/v1.28.yml create mode 100644 roles/container-engine/cri-o/vars/v1.29.yml diff --git a/README.md b/README.md index 2565680a713..897e58505aa 100644 --- a/README.md +++ b/README.md @@ -160,11 +160,11 @@ Note: Upstart/SysV init based OS types are not supported. ## Supported Components - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.6 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.1 - [etcd](https://github.com/etcd-io/etcd) v3.5.10 - [docker](https://www.docker.com/) v20.10 (see note) - [containerd](https://containerd.io/) v1.7.11 - - [cri-o](http://cri-o.io/) v1.27 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) + - [cri-o](http://cri-o.io/) v1.29.1 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0 - [calico](https://github.com/projectcalico/calico) v3.26.4 diff --git a/docs/hardening.md b/docs/hardening.md index fe2f3a568a8..8623bdc30e5 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -97,7 +97,7 @@ kubelet_event_record_qps: 1 kubelet_rotate_certificates: true kubelet_streaming_connection_idle_timeout: "5m" kubelet_make_iptables_util_chains: true -kubelet_feature_gates: ["RotateKubeletServerCertificate=true", "SeccompDefault=true"] +kubelet_feature_gates: ["RotateKubeletServerCertificate=true"] kubelet_seccomp_default: true kubelet_systemd_hardening: true # In case you have multiple interfaces in your diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 68233b4c8cb..37ef5e2caa1 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.28.6 +kube_version: v1.29.1 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml index f20108f86bf..6bc0d72380e 100644 --- a/roles/container-engine/cri-o/defaults/main.yml +++ b/roles/container-engine/cri-o/defaults/main.yml @@ -78,13 +78,6 @@ crio_subuid_length: 16777216 crio_subgid_start: 2130706432 crio_subgid_length: 16777216 -# cri-o binary files -crio_bin_files: - - conmon - - crio - - crio-status - - pinns - # cri-o manual files crio_man_files: 5: diff --git a/roles/container-engine/cri-o/tasks/load_vars.yml b/roles/container-engine/cri-o/tasks/load_vars.yml new file mode 100644 index 00000000000..8a5daa392bb --- /dev/null +++ b/roles/container-engine/cri-o/tasks/load_vars.yml @@ -0,0 +1,8 @@ +--- +- name: Cri-o | include vars/v1.28.yml + include_vars: v1.28.yml + when: crio_version is version("v1.29.0", operator="<") + +- name: Cri-o | include vars/v1.29.yml + include_vars: v1.29.yml + when: crio_version is version("v1.29.0", operator=">=") diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml index 60c818a55f7..99a3ed278ca 100644 --- a/roles/container-engine/cri-o/tasks/main.yaml +++ b/roles/container-engine/cri-o/tasks/main.yaml @@ -1,4 +1,7 @@ --- +- name: Cri-o | load vars + import_tasks: load_vars.yml + - name: Cri-o | check if fedora coreos stat: path: /run/ostree-booted @@ -203,7 +206,7 @@ - not service_start.changed - name: Cri-o | verify that crio is running - command: "{{ bin_dir }}/crio-status info" + command: "{{ bin_dir }}/{{ crio_status_command }} info" register: get_crio_info until: get_crio_info is succeeded changed_when: false diff --git a/roles/container-engine/cri-o/tasks/reset.yml b/roles/container-engine/cri-o/tasks/reset.yml index 1bae013acb1..53d47143477 100644 --- a/roles/container-engine/cri-o/tasks/reset.yml +++ b/roles/container-engine/cri-o/tasks/reset.yml @@ -1,4 +1,7 @@ --- +- name: Cri-o | load vars + import_tasks: load_vars.yml + - name: CRI-O | Kubic repo name for debian os family set_fact: crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x', '')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}" diff --git a/roles/container-engine/cri-o/vars/v1.28.yml b/roles/container-engine/cri-o/vars/v1.28.yml new file mode 100644 index 00000000000..a6a8b673da8 --- /dev/null +++ b/roles/container-engine/cri-o/vars/v1.28.yml @@ -0,0 +1,9 @@ +--- +# cri-o binary files +crio_bin_files: + - conmon + - crio + - crio-status + - pinns + +crio_status_command: crio-status diff --git a/roles/container-engine/cri-o/vars/v1.29.yml b/roles/container-engine/cri-o/vars/v1.29.yml new file mode 100644 index 00000000000..bc25e2e56a2 --- /dev/null +++ b/roles/container-engine/cri-o/vars/v1.29.yml @@ -0,0 +1,9 @@ +--- +# cri-o binary files +crio_bin_files: + - conmon + - conmonrs + - crio + - pinns + +crio_status_command: crio status diff --git a/roles/kubernetes/node/tasks/facts.yml b/roles/kubernetes/node/tasks/facts.yml index 156d748d245..0aaa11d60a7 100644 --- a/roles/kubernetes/node/tasks/facts.yml +++ b/roles/kubernetes/node/tasks/facts.yml @@ -18,7 +18,7 @@ when: container_manager == 'crio' block: - name: Look up crio cgroup driver - shell: "set -o pipefail && {{ bin_dir }}/crio-status info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'" + shell: "set -o pipefail && {{ bin_dir }}/{{ crio_status_command }} info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'" args: executable: /bin/bash register: crio_cgroup_driver_result diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index cc8d676f75c..000954c30d4 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -133,9 +133,9 @@ skopeo_version: "v1.13.2" kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" pod_infra_supported_versions: + v1.29: "3.9" v1.28: "3.9" v1.27: "3.9" - v1.26: "3.9" pod_infra_version: "{{ pod_infra_supported_versions[kube_major_version] }}" etcd_supported_versions: @@ -158,9 +158,9 @@ crio_version: "{{ crio_supported_versions[kube_major_version] }}" # Scheduler plugins doesn't build for K8s 1.28 yet scheduler_plugins_supported_versions: + v1.29: 0 v1.28: 0 v1.27: v0.27.8 - v1.26: v0.26.7 scheduler_plugins_version: "{{ scheduler_plugins_supported_versions[kube_major_version] }}" yq_version: "v4.35.2" diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index f7eb5b37345..114668ac5cb 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -16,7 +16,7 @@ kubelet_fail_swap_on: true kubelet_swap_behavior: LimitedSwap ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.28.6 +kube_version: v1.29.1 ## The minimum version working kube_version_min_required: v1.27.0 diff --git a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml index d8dcc1f8e6d..55cbd506374 100644 --- a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml +++ b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml @@ -86,7 +86,7 @@ kubelet_event_record_qps: 1 kubelet_rotate_certificates: true kubelet_streaming_connection_idle_timeout: "5m" kubelet_make_iptables_util_chains: true -kubelet_feature_gates: ["RotateKubeletServerCertificate=true", "SeccompDefault=true"] +kubelet_feature_gates: ["RotateKubeletServerCertificate=true"] kubelet_seccomp_default: true kubelet_systemd_hardening: true # In case you have multiple interfaces in your From 11c01ef600d6866316e49a879afec686b27eda40 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 31 Jan 2024 14:13:17 +0100 Subject: [PATCH 063/103] docs: vagrant-libvirt is tested in CI (#10847) --- docs/vagrant.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/vagrant.md b/docs/vagrant.md index e331c51129f..7d4fd12805a 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -1,8 +1,8 @@ # Vagrant -Assuming you have Vagrant 2.0+ installed with virtualbox (it may work with libvirt/qemu or -vmware, but is untested) you should be able to launch a 3 node Kubernetes -cluster by simply running `vagrant up`. +Assuming you have Vagrant 2.0+ installed with virtualbox or libvirt/qemu +(vmware may work, but is untested) you should be able to launch a 3 node +Kubernetes cluster by simply running `vagrant up`. This will spin up 3 VMs and install kubernetes on them. Once they are completed you can connect to any of them by running `vagrant ssh k8s-[1..3]`. From dce68e6839e84a87b642033aa9a2c02fb81d102b Mon Sep 17 00:00:00 2001 From: Mohamed Omar Zaian Date: Wed, 31 Jan 2024 14:13:26 +0100 Subject: [PATCH 064/103] [feat] Update metrics server to v0.7.0 (#10856) --- roles/kubespray-defaults/defaults/main/download.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 000954c30d4..601fcce2e6d 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -309,7 +309,7 @@ scheduler_plugins_scheduler_image_tag: "{{ scheduler_plugins_version }}" registry_version: "2.8.1" registry_image_repo: "{{ docker_image_repo }}/library/registry" registry_image_tag: "{{ registry_version }}" -metrics_server_version: "v0.6.4" +metrics_server_version: "v0.7.0" metrics_server_image_repo: "{{ kube_image_repo }}/metrics-server/metrics-server" metrics_server_image_tag: "{{ metrics_server_version }}" local_volume_provisioner_version: "v2.5.0" From 8d53c1723cd2bc54c7c6fcc527673431372cffd9 Mon Sep 17 00:00:00 2001 From: Victor Login Date: Thu, 1 Feb 2024 13:28:20 +0200 Subject: [PATCH 065/103] bump coredns version to 1.11.1 (#10719) * update version coredns 1.11.1 * Update roles/kubespray-defaults/defaults/main/download.yml Co-authored-by: Mohamed Omar Zaian --------- Co-authored-by: Mohamed Omar Zaian --- README.md | 2 +- roles/kubespray-defaults/defaults/main/download.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 897e58505aa..235eb54942d 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ Note: Upstart/SysV init based OS types are not supported. - [kube-vip](https://github.com/kube-vip/kube-vip) v0.5.12 - Application - [cert-manager](https://github.com/jetstack/cert-manager) v1.13.2 - - [coredns](https://github.com/coredns/coredns) v1.10.1 + - [coredns](https://github.com/coredns/coredns) v1.11.1 - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.4 - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4 - [argocd](https://argoproj.github.io/) v2.8.4 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 601fcce2e6d..a604aa38a4a 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -287,7 +287,7 @@ haproxy_image_tag: 2.8.2-alpine # Coredns version should be supported by corefile-migration (or at least work with) # bundle with kubeadm; if not 'basic' upgrade can sometimes fail -coredns_version: "{{ 'v1.10.1' if (kube_version is version('v1.27.0', '>=')) else 'v1.9.3' }}" +coredns_version: "{{ 'v1.11.1' if (kube_version is version('v1.29.0', '>=')) else 'v1.10.1' }}" coredns_image_is_namespaced: "{{ (coredns_version is version('v1.7.1', '>=')) }}" coredns_image_repo: "{{ kube_image_repo }}{{ '/coredns/coredns' if (coredns_image_is_namespaced | bool) else '/coredns' }}" From 0dbde7536fe965640888827d69c33788bb33a924 Mon Sep 17 00:00:00 2001 From: kyrie <139965836+KubeKyrie@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:06:08 +0800 Subject: [PATCH 066/103] make containerd 1.7.12 default and upgrade runc to v1.1.11 (#10862) Signed-off-by: KubeKyrie --- README.md | 2 +- .../defaults/main/checksums.yml | 20 +++++++++++++++++++ .../defaults/main/download.yml | 4 ++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 235eb54942d..a5262909209 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ Note: Upstart/SysV init based OS types are not supported. - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.1 - [etcd](https://github.com/etcd-io/etcd) v3.5.10 - [docker](https://www.docker.com/) v20.10 (see note) - - [containerd](https://containerd.io/) v1.7.11 + - [containerd](https://containerd.io/) v1.7.13 - [cri-o](http://cri-o.io/) v1.29.1 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0 diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index d1d18cbe40b..8f26d06a5e3 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -702,6 +702,8 @@ cri_dockerd_archive_checksums: 0.2.5: 0 runc_checksums: arm: + v1.1.12: 0 + v1.1.11: 0 v1.1.10: 0 v1.1.9: 0 v1.1.8: 0 @@ -711,6 +713,8 @@ runc_checksums: v1.1.4: 0 v1.1.3: 0 arm64: + v1.1.12: 879f910a05c95c10c64ad8eb7d5e3aa8e4b30e65587b3d68e009a3565aed5bb8 + v1.1.11: 9f1ee53f06b78cc4a115ca6ae4eec10567999539ce828a22c5351edba043ed12 v1.1.10: 4830afd426bdeacbdf9cb8729524aa2ed51790b8c4b28786995925593708f1c8 v1.1.9: b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0 v1.1.8: 7c22cb618116d1d5216d79e076349f93a672253d564b19928a099c20e4acd658 @@ -720,6 +724,8 @@ runc_checksums: v1.1.4: dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223 v1.1.3: 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f amd64: + v1.1.12: aadeef400b8f05645768c1476d1023f7875b78f52c7ff1967a6dbce236b8cbd8 + v1.1.11: 77ae134de014613c44d25e6310a57a219a7a91155cd47d069a0f22a2cad5caea v1.1.10: 81f73a59be3d122ab484d7dfe9ddc81030f595cc59968f61c113a9a38a2c113a v1.1.9: b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43 v1.1.8: 1d05ed79854efc707841dfc7afbf3b86546fc1d0b3a204435ca921c14af8385b @@ -729,6 +735,8 @@ runc_checksums: v1.1.4: db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce v1.1.3: 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01 ppc64le: + v1.1.12: 4069d1d57724126e116ad6dbd84409082d1b0afee1ee960b17558f146a742bb6 + v1.1.11: e3d1da41f97db1bb7e9a8d96c9092747c14ee53bc9f160048828e63f3a2d0896 v1.1.10: 94a091c06c363e4af7be398dc31fa6e02576d5ecda6de1cbf3a08fe8662bf678 v1.1.9: 065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6 v1.1.8: a816cd654e804249c4f757cc6bf2aa2c128e4b8e6a993067d44c63c891c081ab @@ -1141,6 +1149,8 @@ nerdctl_archive_checksums: 1.0.0: 2fb02e629a4be16b194bbfc64819132a72ede1f52596bd8e1ec2beaf7c28c117 containerd_archive_checksums: arm: + 1.7.13: 0 + 1.7.12: 0 1.7.11: 0 1.7.10: 0 1.7.9: 0 @@ -1153,6 +1163,7 @@ containerd_archive_checksums: 1.7.2: 0 1.7.1: 0 1.7.0: 0 + 1.6.28: 0 1.6.25: 0 1.6.24: 0 1.6.23: 0 @@ -1185,6 +1196,8 @@ containerd_archive_checksums: 1.5.15: 0 1.5.14: 0 arm64: + 1.7.13: 118759e398f35337109592b4d237538872dc12a207d38832b9d04515d0acbc4d + 1.7.12: 8a1b35a521d071a8828f63fe007a51e5b7ac863a1195f5dee32543b1a9d5f2b6 1.7.11: 5eae27cce38a14be5390d4035127aa11416bc5ae592a9ff25b11870872ce1159 1.7.10: 0667b12a04a896a61cf508a4a77190c280f4a1fa35f38c8a4ba63f605b5ec375 1.7.9: 09ca326dee14e00c439137071747c15cc280480e2c26c1e82698c992dd1889c6 @@ -1197,6 +1210,7 @@ containerd_archive_checksums: 1.7.2: d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6 1.7.1: 1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b 1.7.0: e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2 + 1.6.28: 96a231f875ddf9cc7682b881d408ae993f2bd5d0a40402a74ec4fda672047427 1.6.26: 177bed65b6425255bacbe48d99ea7aa5209d381576962c0962dc8615ef16c5c5 1.6.25: 4948677cfc5f98a1d5d46cec90d6d6f84f6b27cd6d28fd87f7f5936d61580ceb 1.6.24: 1d741e9e2d907f02a8b2a46034a28ff9aacdba88c485cef2f4bad18be9ea23ba @@ -1230,6 +1244,8 @@ containerd_archive_checksums: 1.5.15: 0 1.5.14: 0 amd64: + 1.7.13: c2371c009dd8b7738663333d91e5ab50d204f8bcae24201f45d59060d12c3a23 + 1.7.12: 6a24d8b996533fa1b0d7348fe9813a78cd01fa16cff865a961ad0d556f5cd665 1.7.11: d66161d54546fad502fd50a13fcb79efff033fcd895adc9c44762680dcde4e69 1.7.10: eacb0296bff2ae5225a18492dcb32fb28ad4a1fe0a39ea9073367c7e43dc5838 1.7.9: ccd5b434393666f6ebbc90eea959ffd3e61958a1e3e1cc830a678f040142d4b0 @@ -1242,6 +1258,7 @@ containerd_archive_checksums: 1.7.2: 2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7 1.7.1: 9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85 1.7.0: b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0 + 1.6.28: b2f15c722d1cc8b74ed643068e043b92bd031fc23d53488d1e837cf4b2777391 1.6.26: fa806d3e945a8ad25aa1f8123a98524768ead83af2ed1ab3d922d2dd5fe6b14c 1.6.25: 878b331b5fa65df3d33c68ee355724de0044c25071486086409b374a9c62d145 1.6.24: a56fac5ba03c3d6f74ceae14abdc9fafabcba900105e9890c0ac895cc00164ad @@ -1275,6 +1292,8 @@ containerd_archive_checksums: 1.5.15: 0d09043be08dcf6bf136aa78bfd719e836cf9f9679afa4db0b6e4d478e396528 1.5.14: 8513ead11aca164b7e70bcea0429b4e51dad836b6383b806322e128821aaebbd ppc64le: + 1.7.13: 89605ed2365d5eb779477d11947101236eb44e5244f1e58bb162a9e68d242798 + 1.7.12: 80f16891b387d86712026234de7d4d0365a38106dbe5e51b65b1200b24822721 1.7.11: 6f91c5dabdccd1fc75aae8687381bb185b9eb4200beb29d0993dea8175f5fa61 1.7.10: 15a5191bf7c555956a8565d8786399d51b13f2718d59b1a5b2bd380fc420bf8a 1.7.9: 174b8af2d878ad8410205b9ba44fa8d2a9683a521abf13f168f67b7f7375d5b3 @@ -1287,6 +1306,7 @@ containerd_archive_checksums: 1.7.2: cbe7ec913cb603ca218bd8867efdce4bee3b0e0115e467e51c910467daf8184e 1.7.1: 17d97ef55c6ce7af9778dbafb5e73f577d1b34220043a91cccde49dbcc610342 1.7.0: 051e897d3ee5b8c8097f65be447fea2d29226b583ca5d9ed78e9aebcf4e69889 + 1.6.28: 35411f9d1bafc9cae91c2e30d46a59d01bffc7e18ac7f0942dca9d1d5907ab38 1.6.26: 75fb01a4bd3bcd16263c2f833b8e7081356e2e390dd7eb0710232cb04dac5a01 1.6.25: 3ddcc1739ffeb3e1df786d45518a01d93a5cef243eb6dee61cbdd4cd110bc723 1.6.24: abff9e7ec4cc21d19150d2bc55fc89cf53dc03c002cdaf5016ee82aedead9b03 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index a604aa38a4a..ee54f966752 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -76,11 +76,11 @@ image_arch: "{{ host_architecture | default('amd64') }}" # Versions kubeadm_version: "{{ kube_version }}" crun_version: 1.8.5 -runc_version: v1.1.10 +runc_version: v1.1.12 kata_containers_version: 3.1.3 youki_version: 0.1.0 gvisor_version: 20230807 -containerd_version: 1.7.11 +containerd_version: 1.7.13 cri_dockerd_version: 0.3.4 # this is relevant when container_manager == 'docker' From c9a44e4089bdf291c559f15c855f2cccc04e20f6 Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Mon, 5 Feb 2024 13:55:19 +0800 Subject: [PATCH 067/103] make docker 24.0 default (#10873) Signed-off-by: Kay Yan --- README.md | 4 +- .../container-engine/docker/defaults/main.yml | 2 +- .../docker/vars/debian-bookworm.yml | 48 ------------------- roles/container-engine/docker/vars/debian.yml | 17 ++++--- roles/container-engine/docker/vars/fedora.yml | 17 ++++--- roles/container-engine/docker/vars/kylin.yml | 21 ++++---- .../container-engine/docker/vars/redhat-7.yml | 15 ++++-- roles/container-engine/docker/vars/redhat.yml | 17 ++++--- roles/container-engine/docker/vars/ubuntu.yml | 17 ++++--- .../docker/vars/uniontech.yml | 17 ++++--- .../defaults/main/checksums.yml | 8 ++++ .../defaults/main/download.yml | 4 +- 12 files changed, 91 insertions(+), 96 deletions(-) delete mode 100644 roles/container-engine/docker/vars/debian-bookworm.yml diff --git a/README.md b/README.md index a5262909209..749ec7b50af 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ Note: Upstart/SysV init based OS types are not supported. - Core - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.1 - [etcd](https://github.com/etcd-io/etcd) v3.5.10 - - [docker](https://www.docker.com/) v20.10 (see note) + - [docker](https://www.docker.com/) v24.0 (see [Note](#container-runtime-notes)) - [containerd](https://containerd.io/) v1.7.13 - [cri-o](http://cri-o.io/) v1.29.1 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin @@ -196,7 +196,7 @@ Note: Upstart/SysV init based OS types are not supported. ## Container Runtime Notes -- Supported Docker versions are 18.09, 19.03, 20.10, 23.0 and 24.0. The *recommended* Docker version is 20.10 (except on Debian bookworm which without supporting for 20.10 and below any more). `Kubelet` might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. the YUM ``versionlock`` plugin or ``apt pin``). +- Supported Docker versions are 18.09, 19.03, 20.10, 23.0 and 24.0. The *recommended* Docker version is 24.0. `Kubelet` might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. the YUM ``versionlock`` plugin or ``apt pin``). - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20) ## Requirements diff --git a/roles/container-engine/docker/defaults/main.yml b/roles/container-engine/docker/defaults/main.yml index e537558c34b..f7562086a24 100644 --- a/roles/container-engine/docker/defaults/main.yml +++ b/roles/container-engine/docker/defaults/main.yml @@ -1,5 +1,5 @@ --- -docker_version: '20.10' +docker_version: '24.0' docker_cli_version: "{{ docker_version }}" docker_package_info: diff --git a/roles/container-engine/docker/vars/debian-bookworm.yml b/roles/container-engine/docker/vars/debian-bookworm.yml deleted file mode 100644 index 74a66ccb3e0..00000000000 --- a/roles/container-engine/docker/vars/debian-bookworm.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -docker_version: 24.0 -docker_cli_version: 24.0 -docker_containerd_version: 1.6.21 - -# containerd package info is only relevant for docker -containerd_versioned_pkg: - 'latest': "{{ containerd_package }}" - '1.6.16': "{{ containerd_package }}=1.6.16-1" - '1.6.18': "{{ containerd_package }}=1.6.18-1" - '1.6.19': "{{ containerd_package }}=1.6.19-1" - '1.6.20': "{{ containerd_package }}=1.6.20-1" - '1.6.21': "{{ containerd_package }}=1.6.21-1" - 'stable': "{{ containerd_package }}=1.6.21-1" - 'edge': "{{ containerd_package }}=1.6.21-1" - -# https://download.docker.com/linux/debian/ -docker_versioned_pkg: - 'latest': docker-ce - '23.0': docker-ce=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - '24.0': docker-ce=5:24.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - 'stable': docker-ce=5:24.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - 'edge': docker-ce=5:24.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - -docker_cli_versioned_pkg: - 'latest': docker-ce-cli - '23.0': docker-ce=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - '24.0': docker-ce=5:24.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - 'stable': docker-ce=5:24.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - 'edge': docker-ce=5:24.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} - -docker_package_info: - pkgs: - - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - - "{{ docker_versioned_pkg[docker_version | string] }}" - -docker_repo_key_info: - url: '{{ docker_debian_repo_gpgkey }}' - repo_keys: - - '{{ docker_debian_repo_repokey }}' - -docker_repo_info: - repos: - - > - deb {{ docker_debian_repo_base_url }} - {{ ansible_distribution_release | lower }} - stable diff --git a/roles/container-engine/docker/vars/debian.yml b/roles/container-engine/docker/vars/debian.yml index 9f06004c5f2..02289fa4540 100644 --- a/roles/container-engine/docker/vars/debian.yml +++ b/roles/container-engine/docker/vars/debian.yml @@ -22,8 +22,9 @@ containerd_versioned_pkg: '1.6.15': "{{ containerd_package }}=1.6.15-1" '1.6.16': "{{ containerd_package }}=1.6.16-1" '1.6.18': "{{ containerd_package }}=1.6.18-1" - 'stable': "{{ containerd_package }}=1.6.18-1" - 'edge': "{{ containerd_package }}=1.6.18-1" + '1.6.28': "{{ containerd_package }}=1.6.28-1" + 'stable': "{{ containerd_package }}=1.6.28-1" + 'edge': "{{ containerd_package }}=1.6.28-1" # https://download.docker.com/linux/debian/ docker_versioned_pkg: @@ -31,16 +32,20 @@ docker_versioned_pkg: '18.09': docker-ce=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }} '19.03': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }} '20.10': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} - 'stable': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} - 'edge': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} + '23.0': docker-ce=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli=5:18.09.9~3-0~debian-{{ ansible_distribution_release | lower }} '19.03': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release | lower }} '20.10': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} - 'stable': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} - 'edge': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release | lower }} + '23.0': docker-ce-cli=5:23.0.6-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce-cli=5:24.0.9-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }} docker_package_info: pkgs: diff --git a/roles/container-engine/docker/vars/fedora.yml b/roles/container-engine/docker/vars/fedora.yml index f0b78620a73..39dfbafe52b 100644 --- a/roles/container-engine/docker/vars/fedora.yml +++ b/roles/container-engine/docker/vars/fedora.yml @@ -22,8 +22,9 @@ containerd_versioned_pkg: '1.6.15': "{{ containerd_package }}-1.6.15-3.1.fc{{ ansible_distribution_major_version }}" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.fc{{ ansible_distribution_major_version }}" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.fc{{ ansible_distribution_major_version }}" - 'stable': "{{ containerd_package }}-1.6.18-3.1.fc{{ ansible_distribution_major_version }}" - 'edge': "{{ containerd_package }}-1.6.18-3.1.fc{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.fc{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.fc{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.fc{{ ansible_distribution_major_version }}" # https://docs.docker.com/install/linux/docker-ce/fedora/ # https://download.docker.com/linux/fedora//x86_64/stable/Packages/ @@ -31,15 +32,19 @@ docker_versioned_pkg: 'latest': docker-ce '19.03': docker-ce-19.03.15-3.fc{{ ansible_distribution_major_version }} '20.10': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }} - 'stable': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }} - 'edge': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.fc{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.fc{{ ansible_distribution_major_version }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '19.03': docker-ce-cli-19.03.15-3.fc{{ ansible_distribution_major_version }} '20.10': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }} - 'stable': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }} - 'edge': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.fc{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.fc{{ ansible_distribution_major_version }} docker_package_info: enablerepo: "docker-ce" diff --git a/roles/container-engine/docker/vars/kylin.yml b/roles/container-engine/docker/vars/kylin.yml index b933f151fad..2b8cdf7e66a 100644 --- a/roles/container-engine/docker/vars/kylin.yml +++ b/roles/container-engine/docker/vars/kylin.yml @@ -22,8 +22,9 @@ containerd_versioned_pkg: '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" - 'stable': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" - 'edge': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" docker_version: 19.03 docker_cli_version: 19.03 @@ -33,17 +34,21 @@ docker_cli_version: 19.03 # or do 'yum --showduplicates list docker-engine' docker_versioned_pkg: 'latest': docker-ce - '18.09': docker-ce-3:18.09.9-3.el7 + '18.09': docker-ce-3:18.09.9-3.el{{ ansible_distribution_major_version }} '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} - 'stable': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} - 'edge': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} docker_cli_versioned_pkg: 'latest': docker-ce-cli - '18.09': docker-ce-cli-1:18.09.9-3.el7 + '18.09': docker-ce-cli-1:18.09.9-3.el{{ ansible_distribution_major_version }} '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} - 'stable': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} - 'edge': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} docker_package_info: enablerepo: "docker-ce" diff --git a/roles/container-engine/docker/vars/redhat-7.yml b/roles/container-engine/docker/vars/redhat-7.yml index f50d99d750a..c10e881d97e 100644 --- a/roles/container-engine/docker/vars/redhat-7.yml +++ b/roles/container-engine/docker/vars/redhat-7.yml @@ -22,7 +22,8 @@ containerd_versioned_pkg: '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el7" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el7" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el7" - 'stable': "{{ containerd_package }}-1.6.18-3.1.el7" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el7" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el7" 'edge': "{{ containerd_package }}-1.6.18-3.1.el7" # https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package @@ -33,16 +34,20 @@ docker_versioned_pkg: '18.09': docker-ce-18.09.9-3.el7 '19.03': docker-ce-19.03.15-3.el7 '20.10': docker-ce-20.10.20-3.el7 - 'stable': docker-ce-20.10.20-3.el7 - 'edge': docker-ce-20.10.20-3.el7 + '23.0': docker-ce-23.0.6-1.el7 + '24.0': docker-ce-24.0.9-1.el7 + 'stable': docker-ce-24.0.9-1.el7 + 'edge': docker-ce-24.0.9-1.el7 docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli-18.09.9-3.el7 '19.03': docker-ce-cli-19.03.15-3.el7 '20.10': docker-ce-cli-20.10.20-3.el7 - 'stable': docker-ce-cli-20.10.20-3.el7 - 'edge': docker-ce-cli-20.10.20-3.el7 + '23.0': docker-ce-cli-23.0.6-1.el7 + '24.0': docker-ce-cli-24.0.9-1.el7 + 'stable': docker-ce-cli-24.0.9-1.el7 + 'edge': docker-ce-cli-24.0.9-1.el7 docker_package_info: enablerepo: "docker-ce" diff --git a/roles/container-engine/docker/vars/redhat.yml b/roles/container-engine/docker/vars/redhat.yml index 1de2cbe94a4..453f7f9fadf 100644 --- a/roles/container-engine/docker/vars/redhat.yml +++ b/roles/container-engine/docker/vars/redhat.yml @@ -22,8 +22,9 @@ containerd_versioned_pkg: '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" - 'stable': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" - 'edge': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" # https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package # https://download.docker.com/linux/centos/>/x86_64/stable/Packages/ @@ -33,16 +34,20 @@ docker_versioned_pkg: '18.09': docker-ce-3:18.09.9-3.el7 '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }} - 'stable': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }} - 'edge': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli-1:18.09.9-3.el7 '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }} - 'stable': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }} - 'edge': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} docker_package_info: enablerepo: "docker-ce" diff --git a/roles/container-engine/docker/vars/ubuntu.yml b/roles/container-engine/docker/vars/ubuntu.yml index 313849eeb62..b4c1d289ccd 100644 --- a/roles/container-engine/docker/vars/ubuntu.yml +++ b/roles/container-engine/docker/vars/ubuntu.yml @@ -22,8 +22,9 @@ containerd_versioned_pkg: '1.6.15': "{{ containerd_package }}=1.6.15-1" '1.6.16': "{{ containerd_package }}=1.6.16-1" '1.6.18': "{{ containerd_package }}=1.6.18-1" - 'stable': "{{ containerd_package }}=1.6.18-1" - 'edge': "{{ containerd_package }}=1.6.18-1" + '1.6.28': "{{ containerd_package }}=1.6.28-1" + 'stable': "{{ containerd_package }}=1.6.28-1" + 'edge': "{{ containerd_package }}=1.6.28-1" # https://download.docker.com/linux/ubuntu/ docker_versioned_pkg: @@ -31,16 +32,20 @@ docker_versioned_pkg: '18.09': docker-ce=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }} '19.03': docker-ce=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }} '20.10': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} - 'stable': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} - 'edge': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '23.0': docker-ce=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release | lower }} '19.03': docker-ce-cli=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release | lower }} '20.10': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} - 'stable': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} - 'edge': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release | lower }} + '23.0': docker-ce-cli=5:23.0.6-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + '24.0': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'stable': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} + 'edge': docker-ce-cli=5:24.0.9-1~ubuntu.{{ ansible_distribution_version }}~{{ ansible_distribution_release | lower }} docker_package_info: pkgs: diff --git a/roles/container-engine/docker/vars/uniontech.yml b/roles/container-engine/docker/vars/uniontech.yml index d41cb3bd6d1..0749726d1cf 100644 --- a/roles/container-engine/docker/vars/uniontech.yml +++ b/roles/container-engine/docker/vars/uniontech.yml @@ -20,8 +20,9 @@ containerd_versioned_pkg: '1.6.15': "{{ containerd_package }}-1.6.15-3.1.el{{ ansible_distribution_major_version }}" '1.6.16': "{{ containerd_package }}-1.6.16-3.1.el{{ ansible_distribution_major_version }}" '1.6.18': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" - 'stable': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" - 'edge': "{{ containerd_package }}-1.6.18-3.1.el{{ ansible_distribution_major_version }}" + '1.6.28': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'stable': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" + 'edge': "{{ containerd_package }}-1.6.28-3.1.el{{ ansible_distribution_major_version }}" docker_version: 19.03 docker_cli_version: 19.03 @@ -34,16 +35,20 @@ docker_versioned_pkg: '18.09': docker-ce-3:18.09.9-3.el7 '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }} - 'stable': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }} - 'edge': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-3:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-3:24.0.9-1.el{{ ansible_distribution_major_version }} docker_cli_versioned_pkg: 'latest': docker-ce-cli '18.09': docker-ce-cli-1:18.09.9-3.el7 '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }} '20.10': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }} - 'stable': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }} - 'edge': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }} + '23.0': docker-ce-cli-1:23.0.6-1.el{{ ansible_distribution_major_version }} + '24.0': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'stable': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} + 'edge': docker-ce-cli-1:24.0.9-1.el{{ ansible_distribution_major_version }} docker_package_info: enablerepo: "docker-ce" diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index 8f26d06a5e3..d8ed64c4357 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -657,6 +657,8 @@ helm_archive_checksums: v3.10.3: 93cdf398abc68e388d1b46d49d8e1197544930ecd3e81cc58d0a87a4579d60ed cri_dockerd_archive_checksums: arm: + 0.3.9: 0 + 0.3.8: 0 0.3.7: 0 0.3.6: 0 0.3.5: 0 @@ -668,6 +670,8 @@ cri_dockerd_archive_checksums: 0.2.6: 0 0.2.5: 0 arm64: + 0.3.9: f5051002b4f95b0e8fe7fbd5f8de4493350e010834d2a8b647f2b26c45c6c203 + 0.3.8: 64286af171785f0facb72cf364867600b4db19f43a01db49b8b364f5d04aadae 0.3.7: 8da54563ee7ddee36b1adf1f96b3b7b97ec2bc0ec23559b89d9af8eae5e62d9e 0.3.6: 793b8f57cecf734c47bface10387a8e90994c570b516cb755900f21ebd0a663b 0.3.5: c20014dc5a71e6991a3bd7e1667c744e3807b5675b1724b26bb7c70093582cfe @@ -679,6 +683,8 @@ cri_dockerd_archive_checksums: 0.2.6: 90122641e45e8ff81dbdd4d84c06fd9744b807b87bff5d0db7f826ded326a9fd 0.2.5: 067242bf5e4b39fece10500a239612c7b0723ce9766ba309dbd22acaf1a2def2 amd64: + 0.3.9: a6d9b4b796e9eff830311a2349d259507302cb3955dd07b78296b91e40e8b433 + 0.3.8: e12ea6df8228b7d0794c930d32117c4e5a3dcf25a56c3facdf7006289ec6383c 0.3.7: 518c5d5345085f36d311f274208705d7fdb79337a80c256871ce941d5a7d47a1 0.3.6: cf271d65abee88c0c0a6d9dacb151913bf37d25d45913a7e04b09efe408eae18 0.3.5: 30d47bd89998526d51a8518f9e8ef10baed408ab273879ee0e30350702092938 @@ -690,6 +696,8 @@ cri_dockerd_archive_checksums: 0.2.6: 5d57b160d5a1f75333149823bec3e291a1a0960383ddc9ddd6e4ff177382c755 0.2.5: 1660052586390fd2668421d16265dfcc2bbdba79d923c7ede268cf91935657c1 ppc64le: + 0.3.9: 0 + 0.3.8: 0 0.3.7: 0 0.3.6: 0 0.3.5: 0 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index ee54f966752..2080f43aa4e 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -81,10 +81,10 @@ kata_containers_version: 3.1.3 youki_version: 0.1.0 gvisor_version: 20230807 containerd_version: 1.7.13 -cri_dockerd_version: 0.3.4 +cri_dockerd_version: 0.3.9 # this is relevant when container_manager == 'docker' -docker_containerd_version: 1.6.16 +docker_containerd_version: 1.6.28 # gcr and kubernetes image repo define gcr_image_repo: "gcr.io" From 21289db1811acbe8735d10c23e4cd3b84e0d364d Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 5 Feb 2024 16:45:56 +0100 Subject: [PATCH 068/103] Cleanup main OWNERS file (#10848) Remove inactive reviewers / move inactive approvers to emeritus --- OWNERS_ALIASES | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 231d9a72d95..4834f8ca47f 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -1,32 +1,25 @@ aliases: kubespray-approvers: - - mattymo - - chadswen - - mirwan - - miouge1 - - luckysb - - floryut - - oomichi - cristicalin + - floryut - liupeng0518 - - yankay + - mirwan - mzaian - kubespray-reviewers: - - holmsten - - bozzo - - eppo - oomichi - - jayonlau - - cristicalin - - liupeng0518 - yankay + kubespray-reviewers: - cyclinder - - mzaian - - mrfreezeex - erikjiang + - mrfreezeex + - mzaian - vannten + - yankay kubespray-emeritus_approvers: - - riverzhang - - atoms - ant31 + - atoms + - chadswen + - luckysb + - mattymo + - miouge1 + - riverzhang - woopstar From 200b63031944185473b83a9c47ad56308bc40f24 Mon Sep 17 00:00:00 2001 From: raviranjan Date: Mon, 28 Aug 2023 12:07:03 +0200 Subject: [PATCH 069/103] Adding egress IPv6 for node-local-dns queries --- contrib/terraform/openstack/README.md | 7 ++ contrib/terraform/openstack/kubespray.tf | 7 ++ .../openstack/modules/compute/main.tf | 89 +++++++++++++++++++ .../openstack/modules/compute/variables.tf | 28 ++++++ contrib/terraform/openstack/variables.tf | 49 ++++++++++ 5 files changed, 180 insertions(+) diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index a996692768d..f68e2253138 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -269,11 +269,18 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. |`supplementary_master_groups` | To add ansible groups to the masters, such as `kube_node` for tainting them as nodes, empty by default. | |`supplementary_node_groups` | To add ansible groups to the nodes, such as `kube_ingress` for running ingress controller pods, empty by default. | |`bastion_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, `["0.0.0.0/0"]` by default | +|`bastion_allowed_remote_ipv6_ips` | List of IPv6 CIDR allowed to initiate a SSH connection, `["::/0"]` by default | |`master_allowed_remote_ips` | List of CIDR blocks allowed to initiate an API connection, `["0.0.0.0/0"]` by default | +|`master_allowed_remote_ipv6_ips` | List of IPv6 CIDR blocks allowed to initiate an API connection, `["::/0"]` by default | |`bastion_allowed_ports` | List of ports to open on bastion node, `[]` by default | +|`bastion_allowed_ports_ipv6` | List of ports to open on bastion node for IPv6 CIDR blocks, `[]` by default | |`k8s_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, empty by default | +|`k8s_allowed_remote_ips_ipv6` | List of IPv6 CIDR allowed to initiate a SSH connection, empty by default | +|`k8s_allowed_egress_ipv6_ips` | List of IPv6 CIDRs allowed for egress traffic, `["::/0"]` by default | |`worker_allowed_ports` | List of ports to open on worker nodes, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "0.0.0.0/0"}]` by default | +|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}]` by default | |`master_allowed_ports` | List of ports to open on master nodes, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "0.0.0.0/0"}]`, empty by default | +|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "::/0"}]`, empty by default | |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage | |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage | |`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default | diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf index a1776343298..32c00bd8617 100644 --- a/contrib/terraform/openstack/kubespray.tf +++ b/contrib/terraform/openstack/kubespray.tf @@ -77,14 +77,21 @@ module "compute" { k8s_nodes_fips = module.ips.k8s_nodes_fips bastion_fips = module.ips.bastion_fips bastion_allowed_remote_ips = var.bastion_allowed_remote_ips + bastion_allowed_remote_ipv6_ips = var.bastion_allowed_remote_ipv6_ips master_allowed_remote_ips = var.master_allowed_remote_ips + master_allowed_remote_ipv6_ips = var.master_allowed_remote_ipv6_ips k8s_allowed_remote_ips = var.k8s_allowed_remote_ips + k8s_allowed_remote_ips_ipv6 = var.k8s_allowed_remote_ips_ipv6 k8s_allowed_egress_ips = var.k8s_allowed_egress_ips + k8s_allowed_egress_ipv6_ips = var.k8s_allowed_egress_ipv6_ips supplementary_master_groups = var.supplementary_master_groups supplementary_node_groups = var.supplementary_node_groups master_allowed_ports = var.master_allowed_ports + master_allowed_ports_ipv6 = var.master_allowed_ports_ipv6 worker_allowed_ports = var.worker_allowed_ports + worker_allowed_ports_ipv6 = var.worker_allowed_ports_ipv6 bastion_allowed_ports = var.bastion_allowed_ports + bastion_allowed_ports_ipv6 = var.bastion_allowed_ports_ipv6 use_access_ip = var.use_access_ip master_server_group_policy = var.master_server_group_policy node_server_group_policy = var.node_server_group_policy diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index d161d26f627..2256ea2b4e6 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -70,6 +70,36 @@ resource "openstack_networking_secgroup_rule_v2" "k8s_master_ports" { security_group_id = openstack_networking_secgroup_v2.k8s_master.id } +resource "openstack_networking_secgroup_rule_v2" "k8s_master_ipv6_ingress" { + count = length(var.master_allowed_remote_ipv6_ips) + direction = "ingress" + ethertype = "IPv6" + protocol = "tcp" + port_range_min = "6443" + port_range_max = "6443" + remote_ip_prefix = var.master_allowed_remote_ipv6_ips[count.index] + security_group_id = openstack_networking_secgroup_v2.k8s_master.id +} + +resource "openstack_networking_secgroup_rule_v2" "k8s_master_ports_ipv6_ingress" { + count = length(var.master_allowed_ports_ipv6) + direction = "ingress" + ethertype = "IPv6" + protocol = lookup(var.master_allowed_ports_ipv6[count.index], "protocol", "tcp") + port_range_min = lookup(var.master_allowed_ports_ipv6[count.index], "port_range_min") + port_range_max = lookup(var.master_allowed_ports_ipv6[count.index], "port_range_max") + remote_ip_prefix = lookup(var.master_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0") + security_group_id = openstack_networking_secgroup_v2.k8s_master.id +} + +resource "openstack_networking_secgroup_rule_v2" "master_egress_ipv6" { + count = length(var.k8s_allowed_egress_ipv6_ips) + direction = "egress" + ethertype = "IPv6" + remote_ip_prefix = var.k8s_allowed_egress_ipv6_ips[count.index] + security_group_id = openstack_networking_secgroup_v2.k8s_master.id +} + resource "openstack_networking_secgroup_v2" "bastion" { name = "${var.cluster_name}-bastion" count = var.number_of_bastions != "" ? 1 : 0 @@ -99,6 +129,28 @@ resource "openstack_networking_secgroup_rule_v2" "k8s_bastion_ports" { security_group_id = openstack_networking_secgroup_v2.bastion[0].id } +resource "openstack_networking_secgroup_rule_v2" "bastion_ipv6_ingress" { + count = var.number_of_bastions != "" ? length(var.bastion_allowed_remote_ipv6_ips) : 0 + direction = "ingress" + ethertype = "IPv6" + protocol = "tcp" + port_range_min = "22" + port_range_max = "22" + remote_ip_prefix = var.bastion_allowed_remote_ipv6_ips[count.index] + security_group_id = openstack_networking_secgroup_v2.bastion[0].id +} + +resource "openstack_networking_secgroup_rule_v2" "k8s_bastion_ports_ipv6_ingress" { + count = length(var.bastion_allowed_ports_ipv6) + direction = "ingress" + ethertype = "IPv6" + protocol = lookup(var.bastion_allowed_ports_ipv6[count.index], "protocol", "tcp") + port_range_min = lookup(var.bastion_allowed_ports_ipv6[count.index], "port_range_min") + port_range_max = lookup(var.bastion_allowed_ports_ipv6[count.index], "port_range_max") + remote_ip_prefix = lookup(var.bastion_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0") + security_group_id = openstack_networking_secgroup_v2.bastion[0].id +} + resource "openstack_networking_secgroup_v2" "k8s" { name = "${var.cluster_name}-k8s" description = "${var.cluster_name} - Kubernetes" @@ -112,6 +164,13 @@ resource "openstack_networking_secgroup_rule_v2" "k8s" { security_group_id = openstack_networking_secgroup_v2.k8s.id } +resource "openstack_networking_secgroup_rule_v2" "k8s_ipv6" { + direction = "ingress" + ethertype = "IPv6" + remote_group_id = openstack_networking_secgroup_v2.k8s.id + security_group_id = openstack_networking_secgroup_v2.k8s.id +} + resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips" { count = length(var.k8s_allowed_remote_ips) direction = "ingress" @@ -123,6 +182,17 @@ resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips" { security_group_id = openstack_networking_secgroup_v2.k8s.id } +resource "openstack_networking_secgroup_rule_v2" "k8s_allowed_remote_ips_ipv6" { + count = length(var.k8s_allowed_remote_ips_ipv6) + direction = "ingress" + ethertype = "IPv6" + protocol = "tcp" + port_range_min = "22" + port_range_max = "22" + remote_ip_prefix = var.k8s_allowed_remote_ips_ipv6[count.index] + security_group_id = openstack_networking_secgroup_v2.k8s.id +} + resource "openstack_networking_secgroup_rule_v2" "egress" { count = length(var.k8s_allowed_egress_ips) direction = "egress" @@ -131,6 +201,14 @@ resource "openstack_networking_secgroup_rule_v2" "egress" { security_group_id = openstack_networking_secgroup_v2.k8s.id } +resource "openstack_networking_secgroup_rule_v2" "egress_ipv6" { + count = length(var.k8s_allowed_egress_ipv6_ips) + direction = "egress" + ethertype = "IPv6" + remote_ip_prefix = var.k8s_allowed_egress_ipv6_ips[count.index] + security_group_id = openstack_networking_secgroup_v2.k8s.id +} + resource "openstack_networking_secgroup_v2" "worker" { name = "${var.cluster_name}-k8s-worker" description = "${var.cluster_name} - Kubernetes worker nodes" @@ -155,6 +233,17 @@ resource "openstack_networking_secgroup_rule_v2" "worker" { security_group_id = openstack_networking_secgroup_v2.worker.id } +resource "openstack_networking_secgroup_rule_v2" "worker_ipv6_ingress" { + count = length(var.worker_allowed_ports_ipv6) + direction = "ingress" + ethertype = "IPv6" + protocol = lookup(var.worker_allowed_ports_ipv6[count.index], "protocol", "tcp") + port_range_min = lookup(var.worker_allowed_ports_ipv6[count.index], "port_range_min") + port_range_max = lookup(var.worker_allowed_ports_ipv6[count.index], "port_range_max") + remote_ip_prefix = lookup(var.worker_allowed_ports_ipv6[count.index], "remote_ip_prefix", "::/0") + security_group_id = openstack_networking_secgroup_v2.worker.id +} + resource "openstack_compute_servergroup_v2" "k8s_master" { count = var.master_server_group_policy != "" ? 1 : 0 name = "k8s-master-srvgrp" diff --git a/contrib/terraform/openstack/modules/compute/variables.tf b/contrib/terraform/openstack/modules/compute/variables.tf index 1a78f503e1d..006cce9efe8 100644 --- a/contrib/terraform/openstack/modules/compute/variables.tf +++ b/contrib/terraform/openstack/modules/compute/variables.tf @@ -104,18 +104,34 @@ variable "bastion_allowed_remote_ips" { type = list } +variable "bastion_allowed_remote_ipv6_ips" { + type = list +} + variable "master_allowed_remote_ips" { type = list } +variable "master_allowed_remote_ipv6_ips" { + type = list +} + variable "k8s_allowed_remote_ips" { type = list } +variable "k8s_allowed_remote_ips_ipv6" { + type = list +} + variable "k8s_allowed_egress_ips" { type = list } +variable "k8s_allowed_egress_ipv6_ips" { + type = list +} + variable "k8s_masters" { type = map(object({ az = string @@ -172,14 +188,26 @@ variable "master_allowed_ports" { type = list } +variable "master_allowed_ports_ipv6" { + type = list +} + variable "worker_allowed_ports" { type = list } +variable "worker_allowed_ports_ipv6" { + type = list +} + variable "bastion_allowed_ports" { type = list } +variable "bastion_allowed_ports_ipv6" { + type = list +} + variable "use_access_ip" {} variable "master_server_group_policy" { diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf index 4bb6efbfd4e..ef99c77134a 100644 --- a/contrib/terraform/openstack/variables.tf +++ b/contrib/terraform/openstack/variables.tf @@ -220,30 +220,60 @@ variable "bastion_allowed_remote_ips" { default = ["0.0.0.0/0"] } +variable "bastion_allowed_remote_ipv6_ips" { + description = "An array of IPv6 CIDRs allowed to SSH to hosts" + type = list(string) + default = ["::/0"] +} + variable "master_allowed_remote_ips" { description = "An array of CIDRs allowed to access API of masters" type = list(string) default = ["0.0.0.0/0"] } +variable "master_allowed_remote_ipv6_ips" { + description = "An array of IPv6 CIDRs allowed to access API of masters" + type = list(string) + default = ["::/0"] +} + variable "k8s_allowed_remote_ips" { description = "An array of CIDRs allowed to SSH to hosts" type = list(string) default = [] } +variable "k8s_allowed_remote_ips_ipv6" { + description = "An array of IPv6 CIDRs allowed to SSH to hosts" + type = list(string) + default = [] +} + variable "k8s_allowed_egress_ips" { description = "An array of CIDRs allowed for egress traffic" type = list(string) default = ["0.0.0.0/0"] } +variable "k8s_allowed_egress_ipv6_ips" { + description = "An array of CIDRs allowed for egress IPv6 traffic" + type = list(string) + default = ["::/0"] +} + variable "master_allowed_ports" { type = list(any) default = [] } +variable "master_allowed_ports_ipv6" { + type = list(any) + + default = [] +} + variable "worker_allowed_ports" { type = list(any) @@ -257,12 +287,31 @@ variable "worker_allowed_ports" { ] } +variable "worker_allowed_ports_ipv6" { + type = list(any) + + default = [ + { + "protocol" = "tcp" + "port_range_min" = 30000 + "port_range_max" = 32767 + "remote_ip_prefix" = "::/0" + }, + ] +} + variable "bastion_allowed_ports" { type = list(any) default = [] } +variable "bastion_allowed_ports_ipv6" { + type = list(any) + + default = [] +} + variable "use_access_ip" { default = 1 } From f2bdd4bb2fe7983b1df75dd840254e7ea74a648e Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 5 Feb 2024 16:58:55 +0100 Subject: [PATCH 070/103] Fix logical error when checking for boostrap-os (#10867) Also remove some clutter along the way. --- roles/kubespray-defaults/tasks/main.yaml | 38 ++++++++++-------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/roles/kubespray-defaults/tasks/main.yaml b/roles/kubespray-defaults/tasks/main.yaml index e3defd27a8d..0de87f15096 100644 --- a/roles/kubespray-defaults/tasks/main.yaml +++ b/roles/kubespray-defaults/tasks/main.yaml @@ -1,30 +1,22 @@ --- -- name: Configure defaults - debug: - msg: "Check roles/kubespray-defaults/defaults/main/main.yml" +- name: Set facts variables + # do not run gather facts when bootstrap-os in roles + when: > + ansible_play_role_names | + intersect(['bootstrap-os', 'kubernetes-sigs.kubespray.bootstrap-os']) | + length == 0 tags: - always + block: + - name: Set fallback_ips + import_tasks: fallback_ips.yml + when: fallback_ips is not defined -# do not run gather facts when bootstrap-os in roles -- name: Set fallback_ips - import_tasks: fallback_ips.yml - when: - - "'bootstrap-os' not in ansible_play_role_names or - 'kubernetes-sigs.kubespray.bootstrap-os' not in ansible_play_role_names" - - fallback_ips is not defined - tags: - - always - - -- name: Set no_proxy - import_tasks: no_proxy.yml - when: - - "'bootstrap-os' not in ansible_play_role_names or - 'kubernetes-sigs.kubespray.bootstrap-os' not in ansible_play_role_names" - - http_proxy is defined or https_proxy is defined - - no_proxy is not defined - tags: - - always + - name: Set no_proxy + import_tasks: no_proxy.yml + when: + - http_proxy is defined or https_proxy is defined + - no_proxy is not defined # TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled` - name: Set `etcd_deployment_type` to "kubeadm" if `etcd_kubeadm_enabled` is true From 989ba207e9da2e1364f375450561d08af80c8535 Mon Sep 17 00:00:00 2001 From: Gianmarco Mameli <57061995+gianmarco-mameli@users.noreply.github.com> Date: Mon, 5 Feb 2024 16:59:04 +0100 Subject: [PATCH 071/103] task description modified (#10875) --- roles/download/tasks/download_file.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/download/tasks/download_file.yml b/roles/download/tasks/download_file.yml index 98b9487f511..860f93a2cd0 100644 --- a/roles/download/tasks/download_file.yml +++ b/roles/download/tasks/download_file.yml @@ -7,7 +7,7 @@ set_fact: download_force_cache: "{{ true if download_run_once else download_force_cache }}" - - name: Download_file | Starting download of file + - name: Download_file | Show url of file to dowload debug: msg: "{{ download.url }}" run_once: "{{ download_run_once }}" From de4d6a69ee6bcea53b5fcd2092e758ac5fd0eca8 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 6 Feb 2024 04:34:19 +0000 Subject: [PATCH 072/103] Make more fields mandatory in bug report template (#10883) -> to make people actually give us the information we need --- .github/ISSUE_TEMPLATE/bug-report.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yaml b/.github/ISSUE_TEMPLATE/bug-report.yaml index 0a2ceda9a57..7c308b001f6 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yaml +++ b/.github/ISSUE_TEMPLATE/bug-report.yaml @@ -93,17 +93,24 @@ body: attributes: label: Full inventory with variables placeholder: 'ansible -i inventory/sample/inventory.ini all -m debug -a "var=hostvars[inventory_hostname]"' + description: We recommend using snippets services like https://gist.github.com/ etc. + validations: + required: true - type: input id: ansible_command attributes: label: Command used to invoke ansible + validations: + required: true - type: textarea id: ansible_output attributes: label: Output of ansible run description: We recommend using snippets services like https://gist.github.com/ etc. + validations: + required: true - type: textarea id: anything_else From c698790122a67c31bfbf31624dddf930cdb0317e Mon Sep 17 00:00:00 2001 From: anders-elastisys <112394389+anders-elastisys@users.noreply.github.com> Date: Tue, 6 Feb 2024 08:14:22 +0100 Subject: [PATCH 073/103] add nat_outgoing_ipv6 to calico defaults and docs (#10866) --- docs/calico.md | 10 +++++++++- .../sample/group_vars/k8s_cluster/k8s-net-calico.yml | 1 + roles/network_plugin/calico_defaults/defaults/main.yml | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/calico.md b/docs/calico.md index ce9432cafcd..fb9ab06f8ad 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -222,6 +222,14 @@ calico_node_livenessprobe_timeout: 10 calico_node_readinessprobe_timeout: 10 ``` +### Optional : Enable NAT with IPv6 + +To allow outgoing IPv6 traffic going from pods to the Internet, enable the following: + +```yml +nat_outgoing_ipv6: true # NAT outgoing ipv6 (default value: false). +``` + ## Config encapsulation for cross server traffic Calico supports two types of encapsulation: [VXLAN and IP in IP](https://docs.projectcalico.org/v3.11/networking/vxlan-ipip). VXLAN is the more mature implementation and enabled by default, please check your environment if you need *IP in IP* encapsulation. @@ -235,7 +243,7 @@ If you are running your cluster with the default calico settings and are upgradi * perform a manual migration to vxlan before upgrading kubespray (see migrating from IP in IP to VXLAN below) * pin the pre-2.19 settings in your ansible inventory (see IP in IP mode settings below) -**Note:**: Vxlan in ipv6 only supported when kernel >= 3.12. So if your kernel version < 3.12, Please don't set `calico_vxlan_mode_ipv6: vxlanAlways`. More details see [#Issue 6877](https://github.com/projectcalico/calico/issues/6877). +**Note:**: Vxlan in ipv6 only supported when kernel >= 3.12. So if your kernel version < 3.12, Please don't set `calico_vxlan_mode_ipv6: Always`. More details see [#Issue 6877](https://github.com/projectcalico/calico/issues/6877). ### IP in IP mode diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml index cc0499d00a6..c1cc3dd7d73 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml @@ -11,6 +11,7 @@ calico_cni_name: k8s-pod-network # Enables Internet connectivity from containers # nat_outgoing: true +# nat_outgoing_ipv6: false # Enables Calico CNI "host-local" IPAM plugin # calico_ipam_host_local: true diff --git a/roles/network_plugin/calico_defaults/defaults/main.yml b/roles/network_plugin/calico_defaults/defaults/main.yml index 8e716498bd1..8b909f02114 100644 --- a/roles/network_plugin/calico_defaults/defaults/main.yml +++ b/roles/network_plugin/calico_defaults/defaults/main.yml @@ -4,6 +4,7 @@ calico_cni_name: k8s-pod-network # Enables Internet connectivity from containers nat_outgoing: true +nat_outgoing_ipv6: false # add default ippool name calico_pool_name: "default-pool" From 6f419aa18ecaf94807968914728de416e021bd86 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 6 Feb 2024 08:48:29 +0000 Subject: [PATCH 074/103] Revert "implement download mirrors support (#8474)" (#10884) This reverts commit c6e5314fab3ee2e05590b69f578a4fb1ae1903e5. There is no user of the download mirrors support in kubespray, for a long time. --- roles/download/tasks/download_file.yml | 32 +------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/roles/download/tasks/download_file.yml b/roles/download/tasks/download_file.yml index 860f93a2cd0..9773366f018 100644 --- a/roles/download/tasks/download_file.yml +++ b/roles/download/tasks/download_file.yml @@ -53,41 +53,11 @@ - download_force_cache - not download_localhost - # We check a number of mirrors that may hold the file and pick a working one at random - # This task will avoid logging it's parameters to not leak environment passwords in the log - - name: Download_file | Validate mirrors - uri: - url: "{{ mirror }}" - method: HEAD - validate_certs: "{{ download_validate_certs }}" - url_username: "{{ download.username | default(omit) }}" - url_password: "{{ download.password | default(omit) }}" - force_basic_auth: "{{ download.force_basic_auth | default(omit) }}" - delegate_to: "{{ download_delegate if download_force_cache else inventory_hostname }}" - run_once: "{{ download_force_cache }}" - register: uri_result - become: "{{ not download_localhost }}" - until: uri_result is success - retries: "{{ download_retries }}" - delay: "{{ retry_stagger | default(5) }}" - environment: "{{ proxy_env }}" - no_log: "{{ not (unsafe_show_logs | bool) }}" - loop: "{{ download.mirrors | default([download.url]) }}" - loop_control: - loop_var: mirror - ignore_errors: true - - # Ansible 2.9 requires we convert a generator to a list - - name: Download_file | Get the list of working mirrors - set_fact: - valid_mirror_urls: "{{ uri_result.results | selectattr('failed', 'eq', False) | map(attribute='mirror') | list }}" - delegate_to: "{{ download_delegate if download_force_cache else inventory_hostname }}" - # This must always be called, to check if the checksum matches. On no-match the file is re-downloaded. # This task will avoid logging it's parameters to not leak environment passwords in the log - name: Download_file | Download item get_url: - url: "{{ valid_mirror_urls | random }}" + url: "{{ download.url }}" dest: "{{ file_path_cached if download_force_cache else download.dest }}" owner: "{{ omit if download_localhost else (download.owner | default(omit)) }}" mode: "{{ omit if download_localhost else (download.mode | default(omit)) }}" From 65e22481c69034891e7d24d1d716539de0b83d90 Mon Sep 17 00:00:00 2001 From: Oliver Larsson Date: Tue, 6 Feb 2024 10:11:00 +0100 Subject: [PATCH 075/103] Remove documentation for removed in-tree openstack provider (#10889) --- docs/openstack.md | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/docs/openstack.md b/docs/openstack.md index 0e37a6c818e..1184fc40ce1 100644 --- a/docs/openstack.md +++ b/docs/openstack.md @@ -20,27 +20,11 @@ Kubespray has been tested on a number of OpenStack Public Clouds including (in a - [VexxHost](https://vexxhost.com/) - [Zetta](https://www.zetta.io/) -## The in-tree cloud provider +## The OpenStack cloud provider -To deploy Kubespray on [OpenStack](https://www.openstack.org/) uncomment the `cloud_provider` option in `group_vars/all/all.yml` and set it to `openstack`. +The cloud provider is configured to have Octavia by default in Kubespray. -After that make sure to source in your OpenStack credentials like you would do when using `nova-client` or `neutron-client` by using `source path/to/your/openstack-rc` or `. path/to/your/openstack-rc`. - -For those who prefer to pass the OpenStack CA certificate as a string, one can -base64 encode the cacert file and store it in the variable `openstack_cacert`. - -The next step is to make sure the hostnames in your `inventory` file are identical to your instance names in OpenStack. -Otherwise [cinder](https://wiki.openstack.org/wiki/Cinder) won't work as expected. - -Unless you are using calico or kube-router you can now run the playbook. - -## The external cloud provider - -The in-tree cloud provider is deprecated and will be removed in a future version of Kubernetes. The target release for removing all remaining in-tree cloud providers is set to 1.21. - -The new cloud provider is configured to have Octavia by default in Kubespray. - -- Enable the new external cloud provider in `group_vars/all/all.yml`: +- Enable the external OpenStack cloud provider in `group_vars/all/all.yml`: ```yaml cloud_provider: external From ad565ad9225c7b2fde6188d600eff71249752007 Mon Sep 17 00:00:00 2001 From: flxbwr <136713287+flxbwr@users.noreply.github.com> Date: Tue, 6 Feb 2024 13:58:59 +0300 Subject: [PATCH 076/103] Fix waiting for MetalLB controller (#10858) The current state waiting method is bad to implement. When changing the deployment version, which is execute with the upgrade_cluster in the previous ansible task: "Kubernetes Apps | Install and configure MetalLB", next ansible task: "Kubernetes Apps | Wait for MetalLB controller to be running" may fall with an error. --- roles/kubernetes-apps/metallb/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/metallb/tasks/main.yml b/roles/kubernetes-apps/metallb/tasks/main.yml index eb554c5c212..221fe3c79a3 100644 --- a/roles/kubernetes-apps/metallb/tasks/main.yml +++ b/roles/kubernetes-apps/metallb/tasks/main.yml @@ -33,7 +33,7 @@ - inventory_hostname == groups['kube_control_plane'][0] - name: Kubernetes Apps | Wait for MetalLB controller to be running - command: "{{ bin_dir }}/kubectl -n metallb-system wait --for=condition=ready pod -l app=metallb,component=controller --timeout=2m" + command: "{{ bin_dir }}/kubectl rollout status -n metallb-system deployment -l app=metallb,component=controller --timeout=2m" become: true when: - inventory_hostname == groups['kube_control_plane'][0] From 4b0a134bc9e0683d6f46b87e827e66dde5d8f302 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 8 Feb 2024 10:14:45 +0000 Subject: [PATCH 077/103] Only download kubeadm images where needed (#10899) * Refactor of kubeadm images listing Instead of setting multiples facts, we directly create the dict we need from kubeadm output. * Remove useless 'default' filters in roles/download * Only download kubeadm images where needed --- galaxy.yml | 1 + roles/download/tasks/main.yml | 42 ++++++++++++++++++-- roles/download/tasks/prep_kubeadm_images.yml | 33 ++------------- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index 33b259d5f7f..37f9b6b97fc 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -11,6 +11,7 @@ tags: repository: https://github.com/kubernetes-sigs/kubespray dependencies: ansible.utils: '>=2.5.0' + community.general: '>=3.0.0' build_ignore: - .github - '*.tar.gz' diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 3309ab88e1b..fe83e6c8de0 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -2,7 +2,7 @@ - name: Download | Prepare working directories and variables import_tasks: prep_download.yml when: - - not skip_downloads | default(false) + - not skip_downloads tags: - download - upload @@ -10,7 +10,7 @@ - name: Download | Get kubeadm binary and list of required images include_tasks: prep_kubeadm_images.yml when: - - not skip_downloads | default(false) + - not skip_downloads - inventory_hostname in groups['kube_control_plane'] tags: - download @@ -22,8 +22,44 @@ vars: download: "{{ download_defaults | combine(item.value) }}" include_file: "download_{% if download.container %}container{% else %}file{% endif %}.yml" + kubeadm_images: "{{ skip_kubeadm_images | ternary({}, _kubeadm_images) }}" + # The trick (converting list of tuples to list of dicts) below come from + # https://docs.ansible.com/ansible/latest/collections/community/general/dict_filter.html#examples + _kubeadm_images: "{{ dict(names | map('regex_replace', '^(.*)', 'kubeadm_\\1') | + zip( repos | zip(_tags, _groups) | + map('zip', keys) | map('map', 'reverse') | map('community.general.dict') | + map('combine', defaults))) | + dict2items | rejectattr('key', 'in', excluded) | items2dict }}" + keys: + - repo + - tag + - groups + images: "{{ kubeadm_images_raw.stdout_lines | map('split', ':') }}" + _tags: "{{ images | map(attribute=1) }}" + repos: "{{ images | map(attribute=0) }}" + names: "{{ repos | map('split', '/') | map(attribute=-1) }}" + _groups: "{{ names | map('extract', images_groups) }}" + defaults: + enabled: true + container: true + excluded: + - kubeadm_coredns + - kubeadm_pause + images_groups: + coredns: [] + pause: [] + kube-proxy: + - k8s_cluster + etcd: + - etcd + kube-scheduler: + - kube_control_plane + kube-controller-manager: + - kube_control_plane + kube-apiserver: + - kube_control_plane when: - - not skip_downloads | default(false) + - not skip_downloads - download.enabled - item.value.enabled - (not (item.value.container | default(false))) or (item.value.container and download_container) diff --git a/roles/download/tasks/prep_kubeadm_images.yml b/roles/download/tasks/prep_kubeadm_images.yml index fdfed1d0886..67ac2f72101 100644 --- a/roles/download/tasks/prep_kubeadm_images.yml +++ b/roles/download/tasks/prep_kubeadm_images.yml @@ -20,7 +20,7 @@ dest: "{{ kube_config_dir }}/kubeadm-images.yaml" mode: 0644 when: - - not skip_kubeadm_images | default(false) + - not skip_kubeadm_images - name: Prep_kubeadm_images | Copy kubeadm binary from download dir to system path copy: @@ -36,36 +36,9 @@ state: file - name: Prep_kubeadm_images | Generate list of required images - shell: "set -o pipefail && {{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml | grep -Ev 'coredns|pause'" - args: - executable: /bin/bash + command: "{{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml" register: kubeadm_images_raw run_once: true changed_when: false when: - - not skip_kubeadm_images | default(false) - -- name: Prep_kubeadm_images | Parse list of images - vars: - kubeadm_images_list: "{{ kubeadm_images_raw.stdout_lines }}" - set_fact: - kubeadm_image: - key: "kubeadm_{{ (item | regex_replace('^(?:.*\\/)*', '')).split(':')[0] }}" - value: - enabled: true - container: true - repo: "{{ item | regex_replace('^(.*):.*$', '\\1') }}" - tag: "{{ item | regex_replace('^.*:(.*)$', '\\1') }}" - groups: k8s_cluster - loop: "{{ kubeadm_images_list | flatten(levels=1) }}" - register: kubeadm_images_cooked - run_once: true - when: - - not skip_kubeadm_images | default(false) - -- name: Prep_kubeadm_images | Convert list of images to dict for later use - set_fact: - kubeadm_images: "{{ kubeadm_images_cooked.results | map(attribute='ansible_facts.kubeadm_image') | list | items2dict }}" - run_once: true - when: - - not skip_kubeadm_images | default(false) + - not skip_kubeadm_images From 9648300994812446cf07eb6d8d84a86f5621e05c Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 8 Feb 2024 10:52:10 +0000 Subject: [PATCH 078/103] Add test for coredns_dual in CI (#10903) --- tests/files/packet_debian12-calico.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/files/packet_debian12-calico.yml b/tests/files/packet_debian12-calico.yml index a4adafc84a5..4896d3ba337 100644 --- a/tests/files/packet_debian12-calico.yml +++ b/tests/files/packet_debian12-calico.yml @@ -2,3 +2,7 @@ # Instance settings cloud_image: debian-12 mode: default + +# Kubespray settings + +dns_mode: coredns_dual From ef7197f925ab3dcf714acc2c825e0e968c6c938e Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 9 Feb 2024 10:34:28 +0000 Subject: [PATCH 079/103] Use run_once for checking ansible version instead of localhost (#10908) The `assert` module does not connect to the host anyway, and this avoids skipping the assert when running with --limit. --- playbooks/ansible_version.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/ansible_version.yml b/playbooks/ansible_version.yml index 1fea52356ef..d5a83bd08d7 100644 --- a/playbooks/ansible_version.yml +++ b/playbooks/ansible_version.yml @@ -1,12 +1,12 @@ --- - name: Check Ansible version - hosts: localhost + hosts: all gather_facts: false become: no + run_once: true vars: minimal_ansible_version: 2.15.5 # 2.15 versions before 2.15.5 are known to be buggy for kubespray maximal_ansible_version: 2.17.0 - ansible_connection: local tags: always tasks: - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}" From ad9f194c247e7e00a2d6336ecf3b54e6f4915d34 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 9 Feb 2024 13:40:19 +0000 Subject: [PATCH 080/103] Adjust galaxy.yml (#10906) --- galaxy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 37f9b6b97fc..da5a89a44fb 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -5,10 +5,11 @@ name: kubespray version: 2.25.0 readme: README.md authors: - - luksi1 + - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray) tags: - infrastructure repository: https://github.com/kubernetes-sigs/kubespray +license_file: LICENSE dependencies: ansible.utils: '>=2.5.0' community.general: '>=3.0.0' From f5474ec6ccdc11f51a5f2a8f30ba6b27e8237eda Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 9 Feb 2024 14:04:28 +0000 Subject: [PATCH 081/103] Don't try to set permissions recursively on cache+staging directory (#10900) This should avoid permissions problems when the user creating the directory and the user creating the content are different (when containers images are saved by root for instances, because the user can't use the container runtime). --- roles/download/tasks/prep_download.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/download/tasks/prep_download.yml b/roles/download/tasks/prep_download.yml index 4c737e8e63e..58e50849161 100644 --- a/roles/download/tasks/prep_download.yml +++ b/roles/download/tasks/prep_download.yml @@ -69,7 +69,6 @@ file: path: "{{ local_release_dir }}/images" state: directory - recurse: yes mode: 0755 owner: "{{ ansible_ssh_user | default(ansible_user_id) }}" when: @@ -79,7 +78,6 @@ file: path: "{{ download_cache_dir }}/images" state: directory - recurse: yes mode: 0755 delegate_to: localhost connection: local From ffda3656d179283031a91d8498700d10df55f1b6 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 9 Feb 2024 14:33:16 +0000 Subject: [PATCH 082/103] Enable containerd 'discard_unpacked_layers' by default (#10905) * containerd: Remove redundant 'default' filters * containerd: enable 'discard_unpacked_layers' by default This should help with containerd disk usage --- .../containerd/defaults/main.yml | 8 ++++++-- .../containerd/templates/config.toml.j2 | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml index fd36c50de06..9ecace92f00 100644 --- a/roles/container-engine/containerd/defaults/main.yml +++ b/roles/container-engine/containerd/defaults/main.yml @@ -7,8 +7,8 @@ containerd_systemd_dir: "/etc/systemd/system/containerd.service.d" # Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242 containerd_oom_score: 0 -# containerd_default_runtime: "runc" -# containerd_snapshotter: "native" +containerd_default_runtime: "runc" +containerd_snapshotter: "overlayfs" containerd_runc_runtime: name: runc @@ -36,6 +36,10 @@ containerd_default_base_runtime_spec_patch: hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}" soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}" +# Can help reduce disk usage +# https://github.com/containerd/containerd/discussions/6295 +containerd_discard_unpacked_layers: true + containerd_base_runtime_specs: cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch, recursive=1) }}" diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index e1d4ab2df41..30773aacc90 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -4,29 +4,30 @@ state = "{{ containerd_state_dir }}" oom_score = {{ containerd_oom_score }} [grpc] - max_recv_message_size = {{ containerd_grpc_max_recv_message_size | default(16777216) }} - max_send_message_size = {{ containerd_grpc_max_send_message_size | default(16777216) }} + max_recv_message_size = {{ containerd_grpc_max_recv_message_size }} + max_send_message_size = {{ containerd_grpc_max_send_message_size }} [debug] - level = "{{ containerd_debug_level | default('info') }}" + level = "{{ containerd_debug_level }}" [metrics] - address = "{{ containerd_metrics_address | default('') }}" - grpc_histogram = {{ containerd_metrics_grpc_histogram | default(false) | lower }} + address = "{{ containerd_metrics_address }}" + grpc_histogram = {{ containerd_metrics_grpc_histogram | lower }} [plugins] [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" max_container_log_line_size = {{ containerd_max_container_log_line_size }} - enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | default(false) | lower }} - enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | default(false) | lower }} + enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | lower }} + enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | lower }} {% if enable_cdi %} enable_cdi = true cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] {% endif %} [plugins."io.containerd.grpc.v1.cri".containerd] - default_runtime_name = "{{ containerd_default_runtime | default('runc') }}" - snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}" + default_runtime_name = "{{ containerd_default_runtime }}" + snapshotter = "{{ containerd_snapshotter }}" + discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] {% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}] From 250b80ee7c7e79c8ea4528c66877a28a7547632d Mon Sep 17 00:00:00 2001 From: Arthur Rahmangulov <48437342+arahmangulov@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:27:55 +0400 Subject: [PATCH 083/103] Fixed typos in inventory/sample/group_vars/k8s_cluster (#10911) --- inventory/sample/group_vars/k8s_cluster/addons.yml | 4 ++-- inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml | 2 +- inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index e41aea67d37..e57f8a72565 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -60,7 +60,7 @@ local_volume_provisioner_enabled: false # CSI Volume Snapshot Controller deployment, set this to true if your CSI is able to manage snapshots # currently, setting cinder_csi_enabled=true would automatically enable the snapshot controller -# Longhorn is an extenal CSI that would also require setting this to true but it is not included in kubespray +# Longhorn is an external CSI that would also require setting this to true but it is not included in kubespray # csi_snapshot_controller_enabled: false # csi snapshot namespace # snapshot_controller_namespace: kube-system @@ -178,7 +178,7 @@ metallb_speaker_enabled: "{{ metallb_enabled }}" # speaker: # nodeselector: # kubernetes.io/os: "linux" -# tollerations: +# tolerations: # - key: "node-role.kubernetes.io/control-plane" # operator: "Equal" # value: "" diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml index c1cc3dd7d73..3ba1acb65ae 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml @@ -89,7 +89,7 @@ calico_pool_blocksize: 26 # bird enable BGP routing, required for ipip and no encapsulation modes # calico_network_backend: vxlan -# IP in IP and VXLAN is mutualy exclusive modes. +# IP in IP and VXLAN is mutually exclusive modes. # set IP in IP encapsulation mode: "Always", "CrossSubnet", "Never" # calico_ipip_mode: 'Never' diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml index a1704844dd7..b3190a2f14a 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml @@ -65,7 +65,7 @@ # `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9 # cilium_enable_legacy_services: false -# Unique ID of the cluster. Must be unique across all conneted clusters and +# Unique ID of the cluster. Must be unique across all connected clusters and # in the range of 1 and 255. Only relevant when building a mesh of clusters. # This value is not defined by default # cilium_cluster_id: @@ -75,7 +75,7 @@ # cilium_deploy_additionally: false # Auto direct nodes routes can be used to advertise pods routes in your cluster -# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`). +# without any tunneling (with `cilium_tunnel_mode` sets to `disabled`). # This works only if you have a L2 connectivity between all your nodes. # You wil also have to specify the variable `cilium_native_routing_cidr` to # make this work. Please refer to the cilium documentation for more From bfbb3f8d33ebb32cd6e924a5b220dd0327281171 Mon Sep 17 00:00:00 2001 From: Kundan Kumar Date: Mon, 12 Feb 2024 18:11:03 +0900 Subject: [PATCH 084/103] updated ingress controller version (#10868) --- README.md | 2 +- docs/ingress_controller/ingress_nginx.md | 2 +- roles/kubespray-defaults/defaults/main/download.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 749ec7b50af..63284aff5ea 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Note: Upstart/SysV init based OS types are not supported. - Application - [cert-manager](https://github.com/jetstack/cert-manager) v1.13.2 - [coredns](https://github.com/coredns/coredns) v1.11.1 - - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.4 + - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.6 - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4 - [argocd](https://argoproj.github.io/) v2.8.4 - [helm](https://helm.sh/) v3.13.1 diff --git a/docs/ingress_controller/ingress_nginx.md b/docs/ingress_controller/ingress_nginx.md index 9359c4eb7d3..a77ae480d27 100644 --- a/docs/ingress_controller/ingress_nginx.md +++ b/docs/ingress_controller/ingress_nginx.md @@ -35,7 +35,7 @@ kubectl create clusterrolebinding cluster-admin-binding \ The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version. ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.40.2/deploy/static/provider/cloud/deploy.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.9.6/deploy/static/provider/cloud/deploy.yaml ``` ### Provider Specific Steps diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 2080f43aa4e..91a333dfb8c 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -324,7 +324,7 @@ rbd_provisioner_image_tag: "{{ rbd_provisioner_version }}" local_path_provisioner_version: "v0.0.24" local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner" local_path_provisioner_image_tag: "{{ local_path_provisioner_version }}" -ingress_nginx_version: "v1.9.4" +ingress_nginx_version: "v1.9.6" ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller" ingress_nginx_controller_image_tag: "{{ ingress_nginx_version }}" ingress_nginx_kube_webhook_certgen_image_repo: "{{ kube_image_repo }}/ingress-nginx/kube-webhook-certgen" From bf42ccee4efe0bb28030ef23b4d6e31669d7ddc4 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 12 Feb 2024 10:58:45 +0000 Subject: [PATCH 085/103] Fix ingress-nginx controller election (#10913) Under the original code, leader election failed for ingress controllers as a result of mismatch between election-id in the controller config, and the resourceName in the relevant rule of role 'ingress-nginx'. This appeared in the controller logs. To fix the issue, a command-line option was added to container execution (--election-id=...). Now, the election-id agrees with the resourceName provided in the role-ingress-nginx.yml file. A comment in that file was changed to reflect the new logic. Co-authored-by: Vasilis Samoladas Co-authored-by: Mohamed Omar Zaian --- .../ds-ingress-nginx-controller.yml.j2 | 1 + .../templates/role-ingress-nginx.yml.j2 | 18 ++++++------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 index 70e4ea0ea5b..7f419350bdc 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 @@ -48,6 +48,7 @@ spec: args: - /nginx-ingress-controller - --configmap=$(POD_NAMESPACE)/ingress-nginx + - --election-id=ingress-controller-leader-{{ ingress_nginx_class }} - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - --annotations-prefix=nginx.ingress.kubernetes.io diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 index 6c4b1c13fa7..695b07fbde0 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 @@ -28,23 +28,17 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. + # Defaults to "", defined in + # ds-ingress-nginx-controller.yml.js + # by a command-line argument. + # + # This is the correct behaviour for ingress-controller + # version 1.8.1 resourceNames: ["ingress-controller-leader-{{ ingress_nginx_class }}"] verbs: ["get", "update"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - resourceNames: ["ingress-controller-leader-{{ ingress_nginx_class }}"] - verbs: ["get", "update"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create"] From b60220c597b9d85ce77ac2300a25d1f59909a8b1 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 16 Feb 2024 09:19:22 +0000 Subject: [PATCH 086/103] Update almalinux-8 base image to 8.9 (#10918) Should fix GPG errors in CI --- .../image-builder/roles/kubevirt-images/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml b/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml index 47d9bee415c..c9f3699c6e1 100644 --- a/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml +++ b/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml @@ -57,8 +57,8 @@ images: almalinux-8: filename: AlmaLinux-8-GenericCloud-latest.x86_64.qcow2 - url: https://repo.almalinux.org/almalinux/8.5/cloud/x86_64/images/AlmaLinux-8-GenericCloud-8.5-20211119.x86_64.qcow2 - checksum: sha256:d629247b12802157be127db53a7fcb484b80fceae9896d750c953a51a8c6688f + url: https://repo.almalinux.org/almalinux/8.9/cloud/x86_64/images/AlmaLinux-8-GenericCloud-8.9-20231128.x86_64.qcow2 + checksum: sha256:a1686bc537bce699b512e3233666f5b8f69ed797ff1ce0af52c17fdc52942621 converted: true tag: "latest" From 8d5091a3f7cd30d6509de6aba2e365b2231627d0 Mon Sep 17 00:00:00 2001 From: Radek Smid Date: Fri, 16 Feb 2024 10:27:46 +0100 Subject: [PATCH 087/103] fix: Kubelet not starting because of non-existent feature gate (#10448) --- docs/openstack.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/openstack.md b/docs/openstack.md index 1184fc40ce1..6f53da53444 100644 --- a/docs/openstack.md +++ b/docs/openstack.md @@ -50,14 +50,6 @@ The cloud provider is configured to have Octavia by default in Kubespray. ``` - If you are using OpenStack loadbalancer(s) replace the `openstack_lbaas_subnet_id` with the new `external_openstack_lbaas_subnet_id`. **Note** The new cloud provider is using Octavia instead of Neutron LBaaS by default! -- Enable 3 feature gates to allow migration of all volumes and storage classes (if you have any feature gates already set just add the 3 listed below): - - ```yaml - kube_feature_gates: - - CSIMigration=true - - CSIMigrationOpenStack=true - - ExpandCSIVolumes=true - ``` - If you are in a case of a multi-nic OpenStack VMs (see [kubernetes/cloud-provider-openstack#407](https://github.com/kubernetes/cloud-provider-openstack/issues/407) and [#6083](https://github.com/kubernetes-sigs/kubespray/issues/6083) for explanation), you should override the default OpenStack networking configuration: From 082ac10fbbfa195cd4a2b975f7c916165e15953a Mon Sep 17 00:00:00 2001 From: Mohamed Omar Zaian Date: Fri, 16 Feb 2024 10:40:58 +0100 Subject: [PATCH 088/103] [kubernetes] Add hashes for kubernetes 1.29.2, 1.28.7, 1.27.11 (#10919) --- README.md | 2 +- .../group_vars/k8s_cluster/k8s-cluster.yml | 2 +- .../defaults/main/checksums.yml | 39 +++++++++++++++++++ .../kubespray-defaults/defaults/main/main.yml | 2 +- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 63284aff5ea..8d30c84deac 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ Note: Upstart/SysV init based OS types are not supported. ## Supported Components - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.1 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.2 - [etcd](https://github.com/etcd-io/etcd) v3.5.10 - [docker](https://www.docker.com/) v24.0 (see [Note](#container-runtime-notes)) - [containerd](https://containerd.io/) v1.7.13 diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 37ef5e2caa1..68f4d0ee6c6 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.29.1 +kube_version: v1.29.2 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index d8ed64c4357..963ec2310e7 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -22,6 +22,7 @@ crictl_checksums: v1.27.0: 3e4301c2d4b561d861970004002fe15d49af907963de06c70d326f2af1f145e0 crio_archive_checksums: arm: + v1.29.2: 0 v1.29.1: 0 v1.29.0: 0 v1.28.2: 0 @@ -30,6 +31,7 @@ crio_archive_checksums: v1.27.1: 0 v1.27.0: 0 arm64: + v1.29.2: 9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a v1.29.1: b6d3b502ba1474da370519bcdb9eefa07bb04a5a8e7073fbb7071d6e96f49411 v1.29.0: 2bf11aeb85362ce4b25a7d9fc17bbe80659013425430e5efb922b4388031a027 v1.28.2: 739923cb744a862039557f23823f4cc12feba121bd26ca3cc01d80cc8aaa1efb @@ -38,6 +40,7 @@ crio_archive_checksums: v1.27.1: ddf601e28dc22d878cdd34549402a236afaa47e0a08f39b09e65bab7034b1b97 v1.27.0: c6615360311bff7fdfe1933e8d5030a2e9926b7196c4e7a07fcb10e51a676272 amd64: + v1.29.2: f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960 v1.29.1: 6ff5af9962bfc079275ecec2b6caf5dbd25bf037fff53e87d67f27c2617432c5 v1.29.0: 79c161d8db8ee7f0f4807d6232283d481ef0c20c514b61289238258f66734ac6 v1.28.2: c8002a622e268b73f8d45b0adbdff9422b832106a23be137fabdc8a233b3f787 @@ -59,6 +62,7 @@ kubelet_checksums: arm: v1.29.1: 0 v1.29.0: 0 + v1.28.7: 0 v1.28.6: 0 v1.28.5: 0 v1.28.4: 0 @@ -66,6 +70,8 @@ kubelet_checksums: v1.28.2: 0 v1.28.1: 0 v1.28.0: 0 + v1.27.11: 0 + v1.27.10: 0 v1.27.9: 0 v1.27.8: 0 v1.27.7: 0 @@ -77,8 +83,10 @@ kubelet_checksums: v1.27.1: 0 v1.27.0: 0 arm64: + v1.29.2: 9b4aa572d4cd51a41b1067161d961423d0d12b120fb636ea887a12a975d4b19a v1.29.1: e46417ab1ceae995f0e00d4177959a36ed34b807829422bc9dda70b263fe5c5d v1.29.0: 0e0e4544c2a0a3475529154b7534d0d58683466efa04a2bb2e763b476db0bb16 + v1.28.7: e2c98b39b0b0745ef3e30febaeb8eaaf31ec721012405bd0dcf25e84026c221e v1.28.6: ee2c060deff330d3338e24aec9734c9e5d5aea4fea1905c0795bccff6997a65e v1.28.5: 28ddb696eb6e076f2a2f59ccaa2e409785a63346e5bda819717c6e0f58297702 v1.28.4: bf203989dd9b3987b8a0d2331dcce6319f834b57df810fafba5a4805d54823ac @@ -86,6 +94,7 @@ kubelet_checksums: v1.28.2: 32269e9ec38c561d028b65c3048ea6a100e1292cbe9e505565222455c8096577 v1.28.1: 9b7fa64b2785da4a38768377961e227f8da629c56a5df43ca1b665dd07b56f3c v1.28.0: 05dd12e35783cab4960e885ec0e7d0e461989b94297e7bea9018ccbd15c4dce9 + v1.27.11: e81987a864fb47afe14f65fa4e93760bc19c424335e0f0540c6c725b727ce22a v1.27.10: 0edadc44ef36be8d8106cad9972360c0477540e2d8c0bbeb38fd97fd1d7801d5 v1.27.9: 8a14bc3739f5ca3b23d08301c2e769ee58c8d1cecb7243b46b1c098ae77effd7 v1.27.8: 71849182ceb018dc084f499ad28b7b1afb7f23e35ccaf8421941dd5dafef0d4c @@ -98,8 +107,10 @@ kubelet_checksums: v1.27.1: dbb09d297d924575654db38ed2fc627e35913c2d4000c34613ac6de4995457d0 v1.27.0: 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157 amd64: + v1.29.2: f71a85039b71fe08f1c063a93d61a1c952dc8f9a8c6be9b13fbdac8f0d9ff960 v1.29.1: 1b1975c58d38be1a99a8bcba4564ac489afd223b0abe9f2ab08bbde89d2412a3 v1.29.0: e1c38137db8d8777eed8813646b59bf4d22d19b9011ab11dc28e2e34f6b80a05 + v1.28.7: 120b1495babc4364f7e16a9d0f8b8e6b6f78316d047e4f6de77b5569b05813c7 v1.28.6: 8506df1f20a5f8bba0592f5a4cf5d0cc541047708e664cb88580735400d0b26f v1.28.5: bf37335da58182783a8c63866ec1f895b4c436e3ed96bdd87fe3f8ae8004ba1d v1.28.4: db2a473b73c3754d4011590f2f0aa877657608499590c6b0f8b40bec96a3e9ba @@ -107,6 +118,7 @@ kubelet_checksums: v1.28.2: 17edb866636f14eceaad58c56eab12af7ab3be3c78400aff9680635d927f1185 v1.28.1: 2bc22332f44f8fcd3fce57879fd873f977949ebd261571fbae31fbb2713a5dd3 v1.28.0: bfb6b977100963f2879a33e5fbaa59a5276ba829a957a6819c936e9c1465f981 + v1.27.11: 2ce92a5d8985b93bd8ffc4f5519cd79bf2f844590aa38228a3d809c5bf5986e0 v1.27.10: 25a34bf98bb8a296ea07f1ebbcb496b1e6b6c6da3247695288a7c99fc8c1be2c v1.27.9: ede60eea3acbac3f35dbb23d7b148f45cf169ebbb20af102d3ce141fc0bac60c v1.27.8: 2e0557b38c5b9a1263eed25a0b84d741453ed9c0c7bd916f80eadaf7edfb7784 @@ -119,8 +131,10 @@ kubelet_checksums: v1.27.1: cb2845fff0ce41c400489393da73925d28fbee54cfeb7834cd4d11e622cbd3a7 v1.27.0: 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29 ppc64le: + v1.29.2: b0eb5e0362a4e153ed1239c65b0abb02b2d9fbbca6846d0bab8b285de8c84fca v1.29.1: 467d2b457205363f53f72081295ea390fc25215b0ccc29dc04c4f82925266067 v1.29.0: 67f09f866d3e4aee8211ce9887ec8bc427b188474a882a7af999fc0fee939028 + v1.28.7: c67277445af9a97a375da3caa6a7dae7bb52fa454deb811c5bc89c2838b3322f v1.28.6: 8f79f40bef88aaedfdf7256de48a972295b0069ae0ddefa90dff3f8690c825ce v1.28.5: ae9fe81804ba67ee81e8a5fe1dc18fe285267764c61f831886a25245a11d8528 v1.28.4: d79c97811fb10c1b1f48b69573f1164f108630631d9dba0d991fe924bd305f20 @@ -128,6 +142,7 @@ kubelet_checksums: v1.28.2: 79f568ac700d29f88d669c6b6a09adb3b726bdd13c10aa0839cbc70b414372e5 v1.28.1: 547fc76f0c1d78352fad841ebeacd387fe48750b2648565dfd49197621622fbb v1.28.0: 22de59965f2d220afa24bf04f4c6d6b65a4bb1cd80756c13381973b1ac3b4578 + v1.27.11: ce4dc48a61399038eff680b952386ed9be471c5af1e43dc461e0fb5339de9356 v1.27.10: c5014bed224347245fadec3d763846ec33ccd7a580d0c4ee19a45a948392f20c v1.27.9: f270051c9b0f36da10a5d27011783be042edd396e8c729709c2396f29b72b6d2 v1.27.8: 2354fdb19b5018cabe43fde1979965686afd3c95b75531e678a0064c4a30b4e9 @@ -141,8 +156,10 @@ kubelet_checksums: v1.27.0: 17c061a9f7919697ac71c151c19337f65b86f59f59441687ac92e977d851c75b kubectl_checksums: arm: + v1.29.2: f1bab202f0ce0c4209af0a977fc3dd4076397b1983544e09942ca4f586dff900 v1.29.1: a4b478cc0e9adaab0c5bb3627c20c5228ea0fe2aeff9e805d611eb3edb761972 v1.29.0: a2388eb458d07ec734e4fa02fd0147456a1922a7d6b8e67a32db9d64a4d7621c + v1.28.7: d0c2e9228aff23bdcc62072ef9cebf5ebb0b14fb8638b6df8f7f6d5220c36bd9 v1.28.6: 2358d98d4970c177a3af0ae1c2398f69922074a961a61cdff6ae4a7f13106dc1 v1.28.5: 0819c9d0ea66a1e20d74d9a455090e1f67fe07d671866be342ab55532203f4bc v1.28.4: 835ef8d72f8dec4493b855ddd8e4163f107053496d923c89c216489a45757df6 @@ -150,6 +167,7 @@ kubectl_checksums: v1.28.2: 6576aa70413ff00c593a07b549b8b9d9e5ef73c42bb39ab4af475e0fdb540613 v1.28.1: eaa05dab1bffb8593d8e5caa612530ee5c914ee2be73429b7ce36c3becad893f v1.28.0: 372c4e7bbe98c7067c4b7820c4a440c931ad77f7cb83d3237b439ca3c14d3d37 + v1.27.11: 11586f333abaf2776e0d2f9e02c71ae5eeff3ad8e629815aa8bb7d7e9a406301 v1.27.10: 4d81649935ec127f9aa21954697f82e0796f61e8e6406fd058b3a8b80e858c8e v1.27.9: 89b76aa415018377f2c5fc33fc4d45f4997cc63677336f1768ee8a11593515ce v1.27.8: 2f2936f950beb3f08ee0e45fbf80d020163829b95aa11c99ec726ee1a922329c @@ -162,8 +180,10 @@ kubectl_checksums: v1.27.1: fe704e355bf2c5f69964cd12772687535a11a5e9ec0baf4f27e0a8fb156bc615 v1.27.0: 288470e3eb89a2f55273d753ce6674dfb00e732f2971428acb964810aa726188 arm64: + v1.29.2: 3507ecb4224cf05ae2151a98d4932253624e7762159936d5347b19fe037655ca v1.29.1: 96d6dc7b2bdcd344ce58d17631c452225de5bbf59b83fd3c89c33c6298fb5d8b v1.29.0: 8f7a4bd6bae900a4ddab12bd1399aa652c0d59ea508f39b910e111d248893ff7 + v1.28.7: 13d547495bdea49b223fe06bffb6d2bef96436634847f759107655aa80fc990e v1.28.6: 0de705659a80c3fef01df43cc0926610fe31482f728b0f992818abd9bdcd2cb9 v1.28.5: f87fe017ae3ccfd93df03bf17edd4089672528107f230563b8c9966909661ef2 v1.28.4: edf1e17b41891ec15d59dd3cc62bcd2cdce4b0fd9c2ee058b0967b17534457d7 @@ -171,6 +191,7 @@ kubectl_checksums: v1.28.2: ea6d89b677a8d9df331a82139bb90d9968131530b94eab26cee561531eff4c53 v1.28.1: 46954a604b784a8b0dc16754cfc3fa26aabca9fd4ffd109cd028bfba99d492f6 v1.28.0: f5484bd9cac66b183c653abed30226b561f537d15346c605cc81d98095f1717c + v1.27.11: d30e1aa873e78eb376ddee3c785aa78c44eddc56ce2ef901dac1ce0c2c4f50b0 v1.27.10: 2e1996379d5a8b132e0606fcd3df3c8689e11882630b75cca3b7135126847871 v1.27.9: bda475539fdeda9d8a85a84b967af361af264d0826c121b23b0b62ee9b00cd2d v1.27.8: 97ed6739e2803e63fd2d9de78be22d5ba6205bb63179a16ec773063526525a8e @@ -183,8 +204,10 @@ kubectl_checksums: v1.27.1: fd3cb8f16e6ed8aee9955b76e3027ac423b6d1cc7356867310d128082e2db916 v1.27.0: f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc amd64: + v1.29.2: 7816d067740f47f949be826ac76943167b7b3a38c4f0c18b902fffa8779a5afa v1.29.1: 69ab3a931e826bf7ac14d38ba7ca637d66a6fcb1ca0e3333a2cafdf15482af9f v1.29.0: 0e03ab096163f61ab610b33f37f55709d3af8e16e4dcc1eb682882ef80f96fd5 + v1.28.7: aff42d3167685e4d8e86fda0ad9c6ce6ec6c047bc24d608041d54717a18192ba v1.28.6: c8351fe0611119fd36634dd3f53eb94ec1a2d43ef9e78b92b4846df5cc7aa7e3 v1.28.5: 2a44c0841b794d85b7819b505da2ff3acd5950bd1bcd956863714acc80653574 v1.28.4: 893c92053adea6edbbd4e959c871f5c21edce416988f968bec565d115383f7b8 @@ -192,6 +215,7 @@ kubectl_checksums: v1.28.2: c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 v1.28.1: e7a7d6f9d06fab38b4128785aa80f65c54f6675a0d2abef655259ddd852274e1 v1.28.0: 4717660fd1466ec72d59000bb1d9f5cdc91fac31d491043ca62b34398e0799ce + v1.27.11: 7ae327978a1edb43700070c86f5fd77215792c6b58a7ea70192647e0da848e29 v1.27.10: bfb219643c28d9842fceae51590776f06987835d93fc3cb9b0149c9111c741ac v1.27.9: d0caae91072297b2915dd65f6ef3055d27646dce821ec67d18da35ba9a8dc85b v1.27.8: 027b3161e99fa0a7fa529e8f17f73ee2c0807c81c721ca7cf307f6b41c17bc57 @@ -204,8 +228,10 @@ kubectl_checksums: v1.27.1: 7fe3a762d926fb068bae32c399880e946e8caf3d903078bea9b169dcd5c17f6d v1.27.0: 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87 ppc64le: + v1.29.2: 382552d15a1aa7ec5a316b2a912e7fbdaaff2f3c714cd38b2b0c6a48b670fed8 v1.29.1: b7780124ccfe9640f3a37d242d31e8dbb252bcd379bd0d7bf3776d15baf15ca3 v1.29.0: ea926d8cf25e2ce982ff5c375da32b51ccbd122b721b1bc4a32f52a9a0d073ab + v1.28.7: 1394cc047551bbecffee7f1c28cccd0f3c9839a72344854362a08e98d6513c18 v1.28.6: 60fdb4386b5499dd6a6e3a369f35eef63c99647f7a0436fdbeb4db8c052d14f6 v1.28.5: 4448a9f95421cbe69726aa4d2967d706bc43466b9c656c7425b55431b1c20dd4 v1.28.4: 816ca2cef39c0d1ac8ad60c05ae6f6ea5c4a0ca33748240bd1f019381244ca23 @@ -213,6 +239,7 @@ kubectl_checksums: v1.28.2: 87cca30846fec99a4fbea122b21e938717b309631bd2220de52049fce30d2e81 v1.28.1: 81b45c27abbdf2be6c5203dfccfd76ded1ac273f9f7672e6dcdf3440aa191324 v1.28.0: 7a9dcb4c75b33b9dac497c1a756b1f12c7c63f86fc0f321452360fbe1a79ce0f + v1.27.11: af736cbdb7ae42e696fa3543e483726c1728c95039b9520797511965caca56b6 v1.27.10: 445928336932248cb104d99919e659696afa60f8dd8513821f92775e893d0dcb v1.27.9: 2464d947370b8902e1245b0a75a4ecf55fe2aeee5bc87f2add7da00b73535a59 v1.27.8: e25a09dea99192ff43ee13af61bfadd7c79eb538dc8e85376b6c590b4d471204 @@ -226,8 +253,10 @@ kubectl_checksums: v1.27.0: daa9f1d4fe3f217de2546bca4ac14601f34b34a25c1f571f1e44eb313aee1385 kubeadm_checksums: arm: + v1.29.2: 0 v1.29.1: 0 v1.29.0: 0 + v1.28.7: 0 v1.28.6: 0 v1.28.5: 0 v1.28.4: 0 @@ -235,6 +264,7 @@ kubeadm_checksums: v1.28.2: 0 v1.28.1: 0 v1.28.0: 0 + v1.27.11: 0 v1.27.10: 0 v1.27.9: 0 v1.27.8: 0 @@ -247,8 +277,10 @@ kubeadm_checksums: v1.27.1: 0 v1.27.0: 0 arm64: + v1.29.2: e05720feb9d2d67eff25b0156a5c22e2de37be2ffab4e1f4d31e8c526fafd0e1 v1.29.1: 3bff8c50c104c45e416cce9991706c6ac46365f0defbcd54f8cf4ace0fa68dcf v1.29.0: bbddee2d46d2e1643ae3623698b45b13aa2e858616d61c642f2f49e5bb14c980 + v1.28.7: f556e49494737f97a15bf15bb4b27d45f8747b477302cdfd22dd61816bc02203 v1.28.6: 4298cad464e92eec19cdf3e6a607a82a1d626ae70fedba7956175152ab983457 v1.28.5: 22bb6b3377204e93d008f33ac4924d77adca1478f1ae3b515c03476ba54f1adc v1.28.4: a4422780020954436b8e76ab1c59b68c5581a54432dd3e566c4709bb40c8d4f9 @@ -256,6 +288,7 @@ kubeadm_checksums: v1.28.2: 010789a94cf512d918ec4a3ef8ec734dea0061d89a8293059ef9101ca1bf6bff v1.28.1: 7d2f68917470a5d66bd2a7d62897f59cb4afaeffb2f26c028afa119acd8c3fc8 v1.28.0: b9b473d2d9136559b19eb465006af77df45c09862cd7ce6673a33aae517ff5ab + v1.27.11: b8452d6c3f1331beb3d5fa42466a9bc96638a76c40980dba9822300f230c0858 v1.27.10: ed0447155a7e967ae23480b06b31b2c0aaa871e7c59dfd82ae25b03a1eccf6e6 v1.27.9: d3d022842b0b8e4661222e8873249f5acafdbef52fd1bfb98152a582352b3c40 v1.27.8: 0d0f5b2781d663d314e785d14361aa5a09cfaf6e1694aa3cc731f4f06342ec13 @@ -268,8 +301,10 @@ kubeadm_checksums: v1.27.1: 024a59cd6fc76784b597c0c1cf300526e856e8c9fefa5fa7948158929b739551 v1.27.0: acd805c6783b678ee0068b9dd8165bbfd879c345fd9c25d6a978dbc965f48544 amd64: + v1.29.2: 2d4e4fa8685bcbfb661cb41050cd4756f50a7aa147f68492d51a99f9cdfd69ac v1.29.1: d4d81d9020b550c896376fb9e0586a9f15a332175890d061619b52b3e9bc6cbd v1.29.0: 629d4630657caace9c819fd3797f4a70c397fbd41a2a7e464a0507dad675d52c + v1.28.7: 8aa005bdf6af43e47fc818b26f4cb9f361aae8ec4390519e8d4033be65fbef2b v1.28.6: bda3eda8d51e8746a42b535b7eab7df52b091a796227c3212dc30909a8f1b431 v1.28.5: 2b54078c5ea9e85b27f162f508e0bf834a2753e52a57e896812ec3dca92fe9cd v1.28.4: b4d2531b7cddf782f59555436bc098485b5fa6c05afccdeecf0d62d21d84f5bd @@ -277,6 +312,7 @@ kubeadm_checksums: v1.28.2: 6a4808230661c69431143db2e200ea2d021c7f1b1085e6353583075471310d00 v1.28.1: 6134dbc92dcb83c3bae1a8030f7bb391419b5d13ea94badd3a79b7ece75b2736 v1.28.0: 12ea68bfef0377ccedc1a7c98a05ea76907decbcf1e1ec858a60a7b9b73211bb + v1.27.11: 31bf446a712fb08190838c35d1f4c93b0f975708c59634a5dc3d8915a241c83e v1.27.10: 23985e958443ac1aabdbeeedc675358abc0638eb580707829fd42b0996a0aae5 v1.27.9: 78dddac376fa2f04116022cb44ed39ccb9cb0104e05c5b21b220d5151e5c0f86 v1.27.8: f8864769b8b2d7a14f53eb983f23317ff14d68ab76aba71e9de17ce84c38d4eb @@ -289,8 +325,10 @@ kubeadm_checksums: v1.27.1: c7d32d698e99b90f877025104cb4a9f3f8c707e99e6817940f260135b6d1ad0a v1.27.0: 78d0e04705a7bdb76a514d60f60c073b16334b15f57ee87f064354ca8a233e80 ppc64le: + v1.29.2: a0f8ffa8cbfa4bb061ff028df2f6dbb31a9527c561d8c0186d679559f9f347b4 v1.29.1: 3ec6d90c05dd8e4c6bb1f42fd2fe0f091d85317efaf47d9baebd9af506b3878b v1.29.0: 4c414a463ed4277e9062c797d1c0435aa7aec2fd1688c5d34e3161c898113cb5 + v1.28.7: 60aec330562326fe1ac4a26fe16053c976467fdbc5811c37a5b4a5c13379cac8 v1.28.6: 71fc8af0f80599a991ece0c31b21ca85f3ce49322941a305048d9287c249446c v1.28.5: a9bf8b18711639d9d002f63cebc22c8df1627737891c640f2229461d19b8c321 v1.28.4: 24e4b42b1d0ec68fc291fcc57fa88ec34b9e8ba758e01639873ef2068222af4a @@ -298,6 +336,7 @@ kubeadm_checksums: v1.28.2: fdc28482a4316c84d61b0997c29c4d4c7b11459af9c654fdee3b4a3031f0fcb7 v1.28.1: 73e06f2b614ed5665951f7c059e225a7b0b31319c64a3f57e146fbe7a77fe54e v1.28.0: 146fe9194486e46accd5054fa93939f9608fdbeefefc4bc68e4c40fb4a84ccc9 + v1.27.11: b7da90f29cae799d96e47aadb1f20b567fbded09d3d5ddcb88d3378098c6a1f3 v1.27.10: c928ad330bae724b1ef9775e07285408727513a024e3d86e3d72e05768859db8 v1.27.9: 92da9084fa9f8b8b55436b61ec3c697ef951b0b0416a3b3a7f0dd0e5e4d8cd88 v1.27.8: d65b972cd661cb28972f0df731f9e5b65d959920275bad5ef44ff94d3bb8331d diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index 114668ac5cb..8966431151d 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -16,7 +16,7 @@ kubelet_fail_swap_on: true kubelet_swap_behavior: LimitedSwap ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.29.1 +kube_version: v1.29.2 ## The minimum version working kube_version_min_required: v1.27.0 From 65b0604db74b36b95ae6e538fdb1a0747d635e78 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Fri, 16 Feb 2024 13:08:43 +0000 Subject: [PATCH 089/103] download: Remove deleted kubeadm config field (#10931) --- roles/download/templates/kubeadm-images.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/download/templates/kubeadm-images.yaml.j2 b/roles/download/templates/kubeadm-images.yaml.j2 index 36154b31a88..452e1644d4f 100644 --- a/roles/download/templates/kubeadm-images.yaml.j2 +++ b/roles/download/templates/kubeadm-images.yaml.j2 @@ -20,6 +20,5 @@ etcd: {% endfor %} {% endif %} dns: - type: CoreDNS imageRepository: {{ coredns_image_repo | regex_replace('/coredns(?!/coredns).*$', '') }} imageTag: {{ coredns_image_tag }} From 8fa5ae1865c803b40b249064da82d15ca9b5bac1 Mon Sep 17 00:00:00 2001 From: anders-elastisys <112394389+anders-elastisys@users.noreply.github.com> Date: Sun, 18 Feb 2024 04:34:29 +0100 Subject: [PATCH 090/103] bin: improve manage-offline-container-images script (#10857) Fixes bug for retrieving images with tags containing image digests. Script now gets images from jobs and cronjobs as well. New env variable DESTINATION_REGISTRY to push to another registry instead of local registry. New env variable IMAGES_FROM_FILE to pull images listed in a file instead of getting images from a running k8s environment. New env variable REGISTRY_PORT to override port (default is 5000). --- .gitignore | 2 + contrib/offline/README.md | 10 ++- .../manage-offline-container-images.sh | 83 ++++++++++++++----- 3 files changed, 68 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index cf3a4f478f8..fa68d5606e9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ **/vagrant_ansible_inventory *.iml temp +contrib/offline/container-images +contrib/offline/container-images.tar.gz contrib/offline/offline-files contrib/offline/offline-files.tar.gz .idea diff --git a/contrib/offline/README.md b/contrib/offline/README.md index a0e560ec8cc..a2f3bfe9016 100644 --- a/contrib/offline/README.md +++ b/contrib/offline/README.md @@ -5,15 +5,17 @@ Container image collecting script for offline deployment This script has two features: - -(1) Get container images from an environment which is deployed online. - +(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE + environment variable to get images from a file (e.g. temp/images.list after running the + ./generate_list.sh script). (2) Deploy local container registry and register the container images to the registry. Step(1) should be done online site as a preparation, then we bring the gotten images to the target offline environment. if images are from a private registry, you need to set `PRIVATE_REGISTRY` environment variable. -Then we will run step(2) for registering the images to local registry. +Then we will run step(2) for registering the images to local registry, or to an existing +registry set by the `DESTINATION_REGISTRY` environment variable. By default, the local registry +will run on port 5000. This can be changed with the `REGISTRY_PORT` environment variable Step(1) can be operated with: diff --git a/contrib/offline/manage-offline-container-images.sh b/contrib/offline/manage-offline-container-images.sh index 2098eea639c..37de5caaef2 100755 --- a/contrib/offline/manage-offline-container-images.sh +++ b/contrib/offline/manage-offline-container-images.sh @@ -12,11 +12,24 @@ RETRY_COUNT=5 function create_container_image_tar() { set -e - IMAGES=$(kubectl describe pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq) - # NOTE: etcd and pause cannot be seen as pods. - # The pause image is used for --pod-infra-container-image option of kubelet. - EXT_IMAGES=$(kubectl cluster-info dump | egrep "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g) - IMAGES="${IMAGES} ${EXT_IMAGES}" + if [ -z "${IMAGES_FROM_FILE}" ]; then + echo "Getting images from current \"$(kubectl config current-context)\"" + + IMAGES=$(mktemp --suffix=-images) + trap 'rm -f "${IMAGES}"' EXIT + + kubectl describe cronjobs,jobs,pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq > "${IMAGES}" + # NOTE: etcd and pause cannot be seen as pods. + # The pause image is used for --pod-infra-container-image option of kubelet. + kubectl cluster-info dump | grep -E "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g >> "${IMAGES}" + else + echo "Getting images from file \"${IMAGES_FROM_FILE}\"" + if [ ! -f "${IMAGES_FROM_FILE}" ]; then + echo "${IMAGES_FROM_FILE} is not a file" + exit 1 + fi + IMAGES=$(realpath $IMAGES_FROM_FILE) + fi rm -f ${IMAGE_TAR_FILE} rm -rf ${IMAGE_DIR} @@ -26,9 +39,9 @@ function create_container_image_tar() { sudo ${runtime} pull registry:latest sudo ${runtime} save -o registry-latest.tar registry:latest - for image in ${IMAGES} + while read -r image do - FILE_NAME="$(echo ${image} | sed s@"/"@"-"@g | sed s/":"/"-"/g)".tar + FILE_NAME="$(echo ${image} | sed s@"/"@"-"@g | sed s/":"/"-"/g | sed -E 's/\@.*//g')".tar set +e for step in $(seq 1 ${RETRY_COUNT}) do @@ -48,18 +61,20 @@ function create_container_image_tar() { # so that these parts will be replaced with Kubespray. # - kube_image_repo: "registry.k8s.io" # - gcr_image_repo: "gcr.io" + # - ghcr_image_repo: "ghcr.io" # - docker_image_repo: "docker.io" # - quay_image_repo: "quay.io" FIRST_PART=$(echo ${image} | awk -F"/" '{print $1}') if [ "${FIRST_PART}" = "registry.k8s.io" ] || [ "${FIRST_PART}" = "gcr.io" ] || + [ "${FIRST_PART}" = "ghcr.io" ] || [ "${FIRST_PART}" = "docker.io" ] || [ "${FIRST_PART}" = "quay.io" ] || [ "${FIRST_PART}" = "${PRIVATE_REGISTRY}" ]; then - image=$(echo ${image} | sed s@"${FIRST_PART}/"@@) + image=$(echo ${image} | sed s@"${FIRST_PART}/"@@ | sed -E 's/\@.*/\n/g') fi echo "${FILE_NAME} ${image}" >> ${IMAGE_LIST} - done + done < "${IMAGES}" cd .. sudo chown ${USER} ${IMAGE_DIR}/* @@ -72,6 +87,16 @@ function create_container_image_tar() { } function register_container_images() { + create_registry=false + REGISTRY_PORT=${REGISTRY_PORT:-"5000"} + + if [ -z "${DESTINATION_REGISTRY}" ]; then + echo "DESTINATION_REGISTRY not set, will create local registry" + create_registry=true + DESTINATION_REGISTRY="$(hostname):${REGISTRY_PORT}" + fi + echo "Images will be pushed to ${DESTINATION_REGISTRY}" + if [ ! -f ${IMAGE_TAR_FILE} ]; then echo "${IMAGE_TAR_FILE} should exist." exit 1 @@ -81,18 +106,17 @@ function register_container_images() { fi # To avoid "http: server gave http response to https client" error. - LOCALHOST_NAME=$(hostname) if [ -d /etc/docker/ ]; then set -e # Ubuntu18.04, RHEL7/CentOS7 cp ${CURRENT_DIR}/docker-daemon.json ${TEMP_DIR}/docker-daemon.json - sed -i s@"HOSTNAME"@"${LOCALHOST_NAME}"@ ${TEMP_DIR}/docker-daemon.json + sed -i s@"HOSTNAME"@"$(hostname)"@ ${TEMP_DIR}/docker-daemon.json sudo cp ${TEMP_DIR}/docker-daemon.json /etc/docker/daemon.json elif [ -d /etc/containers/ ]; then set -e # RHEL8/CentOS8 cp ${CURRENT_DIR}/registries.conf ${TEMP_DIR}/registries.conf - sed -i s@"HOSTNAME"@"${LOCALHOST_NAME}"@ ${TEMP_DIR}/registries.conf + sed -i s@"HOSTNAME"@"$(hostname)"@ ${TEMP_DIR}/registries.conf sudo cp ${TEMP_DIR}/registries.conf /etc/containers/registries.conf else echo "runtime package(docker-ce, podman, nerctl, etc.) should be installed" @@ -100,19 +124,28 @@ function register_container_images() { fi tar -zxvf ${IMAGE_TAR_FILE} - sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar - set +e - sudo ${runtime} container inspect registry >/dev/null 2>&1 - if [ $? -ne 0 ]; then - sudo ${runtime} run --restart=always -d -p 5000:5000 --name registry registry:latest + + if [ "${create_registry}" ]; then + sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar + set +e + + sudo ${runtime} container inspect registry >/dev/null 2>&1 + if [ $? -ne 0 ]; then + sudo ${runtime} run --restart=always -d -p "${REGISTRY_PORT}":"${REGISTRY_PORT}" --name registry registry:latest + fi + set -e fi - set -e while read -r line; do file_name=$(echo ${line} | awk '{print $1}') raw_image=$(echo ${line} | awk '{print $2}') - new_image="${LOCALHOST_NAME}:5000/${raw_image}" - org_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1 | awk '{print $3}') + new_image="${DESTINATION_REGISTRY}/${raw_image}" + load_image=$(sudo ${runtime} load -i ${IMAGE_DIR}/${file_name} | head -n1) + org_image=$(echo "${load_image}" | awk '{print $3}') + # special case for tags containing the digest when using docker or podman as the container runtime + if [ "${org_image}" == "ID:" ]; then + org_image=$(echo "${load_image}" | awk '{print $4}') + fi image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//) if [ -z "${file_name}" ]; then echo "Failed to get file_name for line ${line}" @@ -136,7 +169,7 @@ function register_container_images() { done <<< "$(cat ${IMAGE_LIST})" echo "Succeeded to register container images to local registry." - echo "Please specify ${LOCALHOST_NAME}:5000 for the following options in your inventry:" + echo "Please specify \"${DESTINATION_REGISTRY}\" for the following options in your inventry:" echo "- kube_image_repo" echo "- gcr_image_repo" echo "- docker_image_repo" @@ -161,13 +194,17 @@ elif [ "${OPTION}" == "register" ]; then register_container_images else echo "This script has two features:" - echo "(1) Get container images from an environment which is deployed online." + echo "(1) Get container images from an environment which is deployed online, or set IMAGES_FROM_FILE" + echo " environment variable to get images from a file (e.g. temp/images.list after running the" + echo " ./generate_list.sh script)." echo "(2) Deploy local container registry and register the container images to the registry." echo "" echo "Step(1) should be done online site as a preparation, then we bring" echo "the gotten images to the target offline environment. if images are from" echo "a private registry, you need to set PRIVATE_REGISTRY environment variable." - echo "Then we will run step(2) for registering the images to local registry." + echo "Then we will run step(2) for registering the images to local registry, or to an existing" + echo "registry set by the DESTINATION_REGISTRY environment variable. By default, the local registry" + echo "will run on port 5000. This can be changed with the REGISTRY_PORT environment variable" echo "" echo "${IMAGE_TAR_FILE} is created to contain your container images." echo "Please keep this file and bring it to your offline environment." From 9442f28c60c377812531948f6c22c00a711d1bf8 Mon Sep 17 00:00:00 2001 From: "R. P. Taylor" <1686627+rptaylor@users.noreply.github.com> Date: Sat, 17 Feb 2024 20:17:40 -0800 Subject: [PATCH 091/103] do not disable SELinux surreptitiously (#10920) --- roles/kubernetes/preinstall/tasks/0080-system-configurations.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml index 87fb1766725..8159dcc2339 100644 --- a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml +++ b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml @@ -19,7 +19,6 @@ - ansible_os_family == "RedHat" - "'Amazon' not in ansible_distribution" - slc.stat.exists - changed_when: False tags: - bootstrap-os From c13b21e830c8e91d7446b44ae07213ac660e4abc Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 19 Feb 2024 10:00:26 +0000 Subject: [PATCH 092/103] Explicit private/public nature of `*ip` vars (#10904) --- docs/vars.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/vars.md b/docs/vars.md index 9c9f003998d..4f20f59a9bc 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -34,10 +34,10 @@ Some variables of note include: ## Addressing variables -* *ip* - IP to use for binding services (host var) +* *ip* - IP to use for binding services (host var). This would **usually** be the public ip. * *access_ip* - IP for other hosts to use to connect to. Often required when deploying from a cloud, such as OpenStack or GCE and you have separate - public/floating and private IPs. + public/floating and private IPs. This would **usually** be the private ip. * *ansible_default_ipv4.address* - Not Kubespray-specific, but it is used if ip and access_ip are undefined * *ip6* - IPv6 address to use for binding services. (host var) From a0d2bda742be603454ba28e3858ab066a79ea561 Mon Sep 17 00:00:00 2001 From: ABW <49398549+chrxmvtik@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:47:36 +0100 Subject: [PATCH 093/103] feat/add default ingress-nginx service (#10925) feat/add default ingress-nginx service feat/add default ingress-nginx service feat/add default ingress-nginx service --- .../sample/group_vars/k8s_cluster/addons.yml | 1 + .../ingress_nginx/defaults/main.yml | 1 + .../ingress_nginx/tasks/main.yml | 7 ++++++ .../templates/svc-ingress-nginx.yml.j2 | 24 +++++++++++++++++++ 4 files changed, 33 insertions(+) create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index e57f8a72565..5fc115fc6d8 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -99,6 +99,7 @@ rbd_provisioner_enabled: false # Nginx ingress controller deployment ingress_nginx_enabled: false # ingress_nginx_host_network: false +# ingress_nginx_service_type: LoadBalancer ingress_publish_status_address: "" # ingress_nginx_nodeselector: # kubernetes.io/os: "linux" diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml index 7a5c1348814..c83c8c3ed61 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml @@ -1,6 +1,7 @@ --- ingress_nginx_namespace: "ingress-nginx" ingress_nginx_host_network: false +ingress_nginx_service_type: LoadBalancer ingress_publish_status_address: "" ingress_nginx_nodeselector: kubernetes.io/os: "linux" diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml index b67a17f393b..ac64c82427d 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml @@ -24,6 +24,8 @@ - { name: rolebinding-ingress-nginx, file: rolebinding-ingress-nginx.yml, type: rolebinding } - { name: ingressclass-nginx, file: ingressclass-nginx.yml, type: ingressclass } - { name: ds-ingress-nginx-controller, file: ds-ingress-nginx-controller.yml, type: ds } + ingress_nginx_template_for_service: + - { name: svc-ingress-nginx, file: svc-ingress-nginx.yml, type: svc } ingress_nginx_templates_for_webhook: - { name: admission-webhook-configuration, file: admission-webhook-configuration.yml, type: sa } - { name: sa-admission-webhook, file: sa-admission-webhook.yml, type: sa } @@ -33,6 +35,11 @@ - { name: rolebinding-admission-webhook, file: rolebinding-admission-webhook.yml, type: rolebinding } - { name: admission-webhook-job, file: admission-webhook-job.yml, type: job } +- name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Template list for service + set_fact: + ingress_nginx_templates: "{{ ingress_nginx_templates + ingress_nginx_template_for_service }}" + when: not ingress_nginx_host_network + - name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Templates list for webhook set_fact: ingress_nginx_templates: "{{ ingress_nginx_templates + ingress_nginx_templates_for_webhook }}" diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 new file mode 100644 index 00000000000..115232f01c5 --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 @@ -0,0 +1,24 @@ +{% if not ingress_nginx_host_network %} +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + type: {{ ingress_nginx_service_type }} + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: https + port: 443 + targetPort: 443 + protocol: TCP + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/port-of: ingress-nginx +{% endif %} From b34b7e0385371a382a7d650371f240db3db04bb3 Mon Sep 17 00:00:00 2001 From: Alin Dragomir Date: Mon, 19 Feb 2024 12:52:51 +0200 Subject: [PATCH 094/103] Add new OpenStack Cloud for terraform (#10910) --- contrib/terraform/openstack/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index f68e2253138..0ff6ed360d4 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -24,6 +24,7 @@ most modern installs of OpenStack that support the basic services. - [Ultimum](https://ultimum.io/) - [VexxHost](https://vexxhost.com/) - [Zetta](https://www.zetta.io/) +- [Cloudify](https://www.cloudify.ro/en) ## Approach From 4a259ee3f0ead18158b21d6d937bb568e5ed780e Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 19 Feb 2024 10:53:00 +0000 Subject: [PATCH 095/103] Remove mirwan from approvers (#10930) Since they are no longer in the kubernetes-sigs organization, they can no longer be assigned PR to approve. --- OWNERS_ALIASES | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 4834f8ca47f..19bf120f704 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -3,7 +3,6 @@ aliases: - cristicalin - floryut - liupeng0518 - - mirwan - mzaian - oomichi - yankay From 5d822ad8cb10ba03276218d2a473d19551886bb8 Mon Sep 17 00:00:00 2001 From: Ricky Kwan Date: Mon, 19 Feb 2024 05:58:11 -0500 Subject: [PATCH 096/103] Support overriding cni directory owner (#10929) --- roles/network_plugin/cni/defaults/main.yml | 2 ++ roles/network_plugin/cni/tasks/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 roles/network_plugin/cni/defaults/main.yml diff --git a/roles/network_plugin/cni/defaults/main.yml b/roles/network_plugin/cni/defaults/main.yml new file mode 100644 index 00000000000..5d11edfa380 --- /dev/null +++ b/roles/network_plugin/cni/defaults/main.yml @@ -0,0 +1,2 @@ +--- +cni_bin_owner: "{{ kube_owner }}" diff --git a/roles/network_plugin/cni/tasks/main.yml b/roles/network_plugin/cni/tasks/main.yml index d74f169c62a..bcab4efb909 100644 --- a/roles/network_plugin/cni/tasks/main.yml +++ b/roles/network_plugin/cni/tasks/main.yml @@ -4,7 +4,7 @@ path: /opt/cni/bin state: directory mode: 0755 - owner: "{{ kube_owner }}" + owner: "{{ cni_bin_owner }}" recurse: true - name: CNI | Copy cni plugins @@ -12,5 +12,5 @@ src: "{{ downloads.cni.dest }}" dest: "/opt/cni/bin" mode: 0755 - owner: "{{ kube_owner }}" + owner: "{{ cni_bin_owner }}" remote_src: yes From d40b073f977f78f7b70572ad34441f4e147eaff1 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 19 Feb 2024 10:58:20 +0000 Subject: [PATCH 097/103] Add extra_vars support to vagrant setup (#10932) --- Vagrantfile | 2 ++ docs/vagrant.md | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index fb8f1a7179c..a23c80e4cb2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -80,6 +80,7 @@ $ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || "" $vagrant_dir ||= File.join(File.dirname(__FILE__), ".vagrant") $playbook ||= "cluster.yml" +$extra_vars ||= {} host_vars = {} @@ -276,6 +277,7 @@ Vagrant.configure("2") do |config| ansible.host_key_checking = false ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"] ansible.host_vars = host_vars + ansible.extra_vars = $extra_vars if $ansible_tags != "" ansible.tags = [$ansible_tags] end diff --git a/docs/vagrant.md b/docs/vagrant.md index 7d4fd12805a..9e12351af1c 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -85,6 +85,11 @@ cat << EOF > vagrant/config.rb \$network_plugin = "flannel" \$inventory = "$INV" \$shared_folders = { 'temp/docker_rpms' => "/var/cache/yum/x86_64/7/docker-ce/packages" } +\$extra_vars = { + dns_domain: my.custom.domain +} +# or +\$extra_vars = "path/to/extra/vars/file.yml" EOF # make the rpm cache From 667bb2c913c1d53b62862ec35377277b4a48285e Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 19 Feb 2024 17:46:29 +0000 Subject: [PATCH 098/103] Remove holmsten from approver (contrib/terraform) (#10936) As they're no longer in kubernetes-sigs org. --- contrib/terraform/OWNERS | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/terraform/OWNERS b/contrib/terraform/OWNERS index b58878dd8d0..f752c6e5851 100644 --- a/contrib/terraform/OWNERS +++ b/contrib/terraform/OWNERS @@ -1,5 +1,3 @@ # See the OWNERS docs at https://go.k8s.io/owners - approvers: - - holmsten - miouge1 From 12c8d0456f92027fd58e8dc30d0f032d316efb03 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 19 Feb 2024 18:00:39 +0000 Subject: [PATCH 099/103] Move perma failing jobs in CI to manual run (#10886) Should free CI resources --- .gitlab-ci/molecule.yml | 12 ++++++------ .gitlab-ci/vagrant.yml | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci/molecule.yml b/.gitlab-ci/molecule.yml index 3961be11ad0..94e5a5606fb 100644 --- a/.gitlab-ci/molecule.yml +++ b/.gitlab-ci/molecule.yml @@ -61,23 +61,23 @@ molecule_cri-o: molecule_kata: extends: .molecule stage: deploy-part3 - allow_failure: true script: - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers - when: on_success + when: manual +# FIXME: this test is broken (perma-failing) molecule_gvisor: extends: .molecule stage: deploy-part3 - allow_failure: true script: - ./tests/scripts/molecule_run.sh -i container-engine/gvisor - when: on_success + when: manual +# FIXME: this test is broken (perma-failing) molecule_youki: extends: .molecule stage: deploy-part3 - allow_failure: true script: - ./tests/scripts/molecule_run.sh -i container-engine/youki - when: on_success + when: manual +# FIXME: this test is broken (perma-failing) diff --git a/.gitlab-ci/vagrant.yml b/.gitlab-ci/vagrant.yml index c0250681b89..a02e3983566 100644 --- a/.gitlab-ci/vagrant.yml +++ b/.gitlab-ci/vagrant.yml @@ -18,12 +18,12 @@ - ./tests/scripts/testcases_run.sh after_script: - chronic ./tests/scripts/testcases_cleanup.sh - allow_failure: true vagrant_ubuntu20-calico-dual-stack: stage: deploy-part2 extends: .vagrant - when: on_success + when: manual +# FIXME: this test if broken (perma-failing) vagrant_ubuntu20-weave-medium: stage: deploy-part2 @@ -55,7 +55,8 @@ vagrant_ubuntu20-kube-router-svc-proxy: vagrant_fedora37-kube-router: stage: deploy-part2 extends: .vagrant - when: on_success + when: manual +# FIXME: this test if broken (perma-failing) vagrant_centos7-kube-router: stage: deploy-part2 From e250bb65bb08fb142b4911e00f0873379bd51b06 Mon Sep 17 00:00:00 2001 From: jaszil <146323284+jaszil@users.noreply.github.com> Date: Tue, 20 Feb 2024 02:42:21 +0100 Subject: [PATCH 100/103] [Terraform-Openstack] Add optional Octavia loadbalancer for Master Nodes (#10924) * terraform-openstack add optional master lb * remove unnecessary spaces * remove unnecessary spaces * fix ci warnings --- contrib/terraform/openstack/README.md | 7 ++- contrib/terraform/openstack/kubespray.tf | 18 +++++++ .../openstack/modules/compute/outputs.tf | 3 ++ .../openstack/modules/loadbalancer/main.tf | 54 +++++++++++++++++++ .../modules/loadbalancer/variables.tf | 15 ++++++ .../modules/loadbalancer/versions.tf | 8 +++ contrib/terraform/openstack/variables.tf | 20 +++++++ 7 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 contrib/terraform/openstack/modules/compute/outputs.tf create mode 100644 contrib/terraform/openstack/modules/loadbalancer/main.tf create mode 100644 contrib/terraform/openstack/modules/loadbalancer/variables.tf create mode 100644 contrib/terraform/openstack/modules/loadbalancer/versions.tf diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index 0ff6ed360d4..923aa314074 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -98,9 +98,10 @@ binaries available on hyperkube v1.4.3_coreos.0 or higher. ## Module Architecture -The configuration is divided into three modules: +The configuration is divided into four modules: - Network +- Loadbalancer - IPs - Compute @@ -298,6 +299,10 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. |`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default | |`k8s_nodes` | Map containing worker node definition, see explanation below | |`k8s_masters` | Map containing master node definition, see explanation for k8s_nodes and `sample-inventory/cluster.tfvars` | +| `k8s_master_loadbalancer_enabled`| Enable and use an Octavia load balancer for the K8s master nodes | +| `k8s_master_loadbalancer_listener_port` | Define via which port the K8s Api should be exposed. `6443` by default | +| `k8s_master_loadbalancer_server_port` | Define via which port the K8S api is available on the mas. `6443` by default | +| `k8s_master_loadbalancer_public_ip` | Specify if an existing floating IP should be used for the load balancer. A new floating IP is assigned by default | ##### k8s_nodes diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf index 32c00bd8617..556fa54340f 100644 --- a/contrib/terraform/openstack/kubespray.tf +++ b/contrib/terraform/openstack/kubespray.tf @@ -112,6 +112,24 @@ module "compute" { ] } +module "loadbalancer" { + source = "./modules/loadbalancer" + + cluster_name = var.cluster_name + subnet_id = module.network.subnet_id + floatingip_pool = var.floatingip_pool + k8s_master_ips = module.compute.k8s_master_ips + k8s_master_loadbalancer_enabled = var.k8s_master_loadbalancer_enabled + k8s_master_loadbalancer_listener_port = var.k8s_master_loadbalancer_listener_port + k8s_master_loadbalancer_server_port = var.k8s_master_loadbalancer_server_port + k8s_master_loadbalancer_public_ip = var.k8s_master_loadbalancer_public_ip + + depends_on = [ + module.compute.k8s_master + ] +} + + output "private_subnet_id" { value = module.network.subnet_id } diff --git a/contrib/terraform/openstack/modules/compute/outputs.tf b/contrib/terraform/openstack/modules/compute/outputs.tf new file mode 100644 index 00000000000..741e9f035df --- /dev/null +++ b/contrib/terraform/openstack/modules/compute/outputs.tf @@ -0,0 +1,3 @@ +output "k8s_master_ips" { + value = concat(openstack_compute_instance_v2.k8s_master_no_floating_ip.*, openstack_compute_instance_v2.k8s_master_no_floating_ip_no_etcd.*) +} diff --git a/contrib/terraform/openstack/modules/loadbalancer/main.tf b/contrib/terraform/openstack/modules/loadbalancer/main.tf new file mode 100644 index 00000000000..12fa225ea6e --- /dev/null +++ b/contrib/terraform/openstack/modules/loadbalancer/main.tf @@ -0,0 +1,54 @@ +resource "openstack_lb_loadbalancer_v2" "k8s_lb" { + count = var.k8s_master_loadbalancer_enabled ? 1 : 0 + name = "${var.cluster_name}-api-loadbalancer" + vip_subnet_id = var.subnet_id +} + +resource "openstack_lb_listener_v2" "api_listener"{ + count = var.k8s_master_loadbalancer_enabled ? 1 : 0 + name = "api-listener" + protocol = "TCP" + protocol_port = var.k8s_master_loadbalancer_listener_port + loadbalancer_id = openstack_lb_loadbalancer_v2.k8s_lb[0].id + depends_on = [ openstack_lb_loadbalancer_v2.k8s_lb ] +} + +resource "openstack_lb_pool_v2" "api_pool" { + count = var.k8s_master_loadbalancer_enabled ? 1 : 0 + name = "api-pool" + protocol = "TCP" + lb_method = "ROUND_ROBIN" + listener_id = openstack_lb_listener_v2.api_listener[0].id + depends_on = [ openstack_lb_listener_v2.api_listener ] +} + +resource "openstack_lb_member_v2" "lb_member" { + count = var.k8s_master_loadbalancer_enabled ? length(var.k8s_master_ips) : 0 + name = var.k8s_master_ips[count.index].name + pool_id = openstack_lb_pool_v2.api_pool[0].id + address = var.k8s_master_ips[count.index].access_ip_v4 + protocol_port = var.k8s_master_loadbalancer_server_port + depends_on = [ openstack_lb_pool_v2.api_pool ] +} + +resource "openstack_lb_monitor_v2" "monitor" { + count = var.k8s_master_loadbalancer_enabled ? 1 : 0 + name = "Api Monitor" + pool_id = openstack_lb_pool_v2.api_pool[0].id + type = "TCP" + delay = 10 + timeout = 5 + max_retries = 5 +} + +resource "openstack_networking_floatingip_v2" "floatip_1" { + count = var.k8s_master_loadbalancer_enabled && var.k8s_master_loadbalancer_public_ip == "" ? 1 : 0 + pool = var.floatingip_pool +} + +resource "openstack_networking_floatingip_associate_v2" "public_ip" { + count = var.k8s_master_loadbalancer_enabled ? 1 : 0 + floating_ip = var.k8s_master_loadbalancer_public_ip != "" ? var.k8s_master_loadbalancer_public_ip : openstack_networking_floatingip_v2.floatip_1[0].address + port_id = openstack_lb_loadbalancer_v2.k8s_lb[0].vip_port_id + depends_on = [ openstack_lb_loadbalancer_v2.k8s_lb ] +} diff --git a/contrib/terraform/openstack/modules/loadbalancer/variables.tf b/contrib/terraform/openstack/modules/loadbalancer/variables.tf new file mode 100644 index 00000000000..40b1b588cef --- /dev/null +++ b/contrib/terraform/openstack/modules/loadbalancer/variables.tf @@ -0,0 +1,15 @@ +variable "cluster_name" {} + +variable "subnet_id" {} + +variable "floatingip_pool" {} + +variable "k8s_master_ips" {} + +variable "k8s_master_loadbalancer_enabled" {} + +variable "k8s_master_loadbalancer_listener_port" {} + +variable "k8s_master_loadbalancer_server_port" {} + +variable "k8s_master_loadbalancer_public_ip" {} diff --git a/contrib/terraform/openstack/modules/loadbalancer/versions.tf b/contrib/terraform/openstack/modules/loadbalancer/versions.tf new file mode 100644 index 00000000000..6c942790da8 --- /dev/null +++ b/contrib/terraform/openstack/modules/loadbalancer/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + } + } + required_version = ">= 0.12.26" +} diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf index ef99c77134a..90416df50d0 100644 --- a/contrib/terraform/openstack/variables.tf +++ b/contrib/terraform/openstack/variables.tf @@ -389,3 +389,23 @@ variable "group_vars_path" { type = string default = "./group_vars" } + +variable "k8s_master_loadbalancer_enabled" { + type = bool + default = "false" +} + +variable "k8s_master_loadbalancer_listener_port" { + type = string + default = "6443" +} + +variable "k8s_master_loadbalancer_server_port" { + type = string + default = 6443 +} + +variable "k8s_master_loadbalancer_public_ip" { + type = string + default = "" +} From 26034b296e9b70373bf151a2098f1dd41ea53490 Mon Sep 17 00:00:00 2001 From: Ludovic Logiou Date: Thu, 22 Feb 2024 14:06:40 +0100 Subject: [PATCH 101/103] Bump cinder-csi version and switch container registry (#10894) * Bump cinder-csi version and switch container registry Signed-off-by: Ludovic Logiou * Update roles/kubespray-defaults/defaults/main/download.yml Co-authored-by: Mohamed Omar Zaian --------- Signed-off-by: Ludovic Logiou Co-authored-by: Mohamed Omar Zaian --- README.md | 2 +- inventory/sample/group_vars/all/openstack.yml | 12 ++++++++++++ .../csi_driver/cinder/defaults/main.yml | 7 +++++++ .../templates/cinder-csi-controllerplugin.yml.j2 | 14 ++++++++------ .../templates/cinder-csi-nodeplugin-rbac.yml.j2 | 10 +--------- .../cinder/templates/cinder-csi-nodeplugin.yml.j2 | 2 ++ .../kubespray-defaults/defaults/main/download.yml | 4 ++-- 7 files changed, 33 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8d30c84deac..6e6e7610f59 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Note: Upstart/SysV init based OS types are not supported. - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11 - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) v0.5.0 - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) v1.10.0 - - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.22.0 + - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.29.0 - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) v1.9.2 - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) v0.0.24 - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) v2.5.0 diff --git a/inventory/sample/group_vars/all/openstack.yml b/inventory/sample/group_vars/all/openstack.yml index cac548f84c0..6cc13998c78 100644 --- a/inventory/sample/group_vars/all/openstack.yml +++ b/inventory/sample/group_vars/all/openstack.yml @@ -44,6 +44,18 @@ ## The tag of the external OpenStack Cloud Controller image # external_openstack_cloud_controller_image_tag: "latest" +## Tags for the Cinder CSI images +## registry.k8s.io/sig-storage/csi-attacher +# cinder_csi_attacher_image_tag: "v4.4.2" +## registry.k8s.io/sig-storage/csi-provisioner +# cinder_csi_provisioner_image_tag: "v3.6.2" +## registry.k8s.io/sig-storage/csi-snapshotter +# cinder_csi_snapshotter_image_tag: "v6.3.2" +## registry.k8s.io/sig-storage/csi-resizer +# cinder_csi_resizer_image_tag: "v1.9.2" +## registry.k8s.io/sig-storage/livenessprobe +# cinder_csi_livenessprobe_image_tag: "v2.11.0" + ## To use Cinder CSI plugin to provision volumes set this value to true ## Make sure to source in the openstack credentials # cinder_csi_enabled: true diff --git a/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml b/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml index 501f3689c0a..8776c30643e 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml +++ b/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml @@ -1,4 +1,11 @@ --- + +cinder_csi_attacher_image_tag: "v4.4.2" +cinder_csi_provisioner_image_tag: "v3.6.2" +cinder_csi_snapshotter_image_tag: "v6.3.2" +cinder_csi_resizer_image_tag: "v1.9.2" +cinder_csi_livenessprobe_image_tag: "v2.11.0" + # To access Cinder, the CSI controller will need credentials to access # openstack apis. Per default this values will be # read from the environment. diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 index 4fe7e475c62..de27b767944 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 @@ -1,5 +1,6 @@ # This YAML file contains CSI Controller Plugin Sidecars # external-attacher, external-provisioner, external-snapshotter +# external-resize, liveness-probe --- kind: Deployment @@ -17,10 +18,10 @@ spec: labels: app: csi-cinder-controllerplugin spec: - serviceAccountName: csi-cinder-controller-sa + serviceAccount: csi-cinder-controller-sa containers: - name: csi-attacher - image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} + image: {{ csi_attacher_image_repo }}:{{ cinder_csi_attacher_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" @@ -28,6 +29,7 @@ spec: {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} - --leader-election=true {% endif %} + - "--default-fstype=ext4" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -35,7 +37,7 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-provisioner - image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }} + image: {{ csi_provisioner_image_repo }}:{{ cinder_csi_provisioner_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" @@ -55,7 +57,7 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-snapshotter - image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }} + image: {{ csi_snapshotter_image_repo }}:{{ cinder_csi_snapshotter_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" @@ -71,7 +73,7 @@ spec: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - name: csi-resizer - image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }} + image: {{ csi_resizer_image_repo }}:{{ cinder_csi_resizer_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" @@ -87,7 +89,7 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: liveness-probe - image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + image: {{ csi_livenessprobe_image_repo }}:{{ cinder_csi_livenessprobe_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - "--csi-address=$(ADDRESS)" diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 index db589636d1b..912923fcad6 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 @@ -14,15 +14,7 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] + --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 index d8346cc5b2d..dd2736eea0a 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 @@ -15,6 +15,8 @@ spec: labels: app: csi-cinder-nodeplugin spec: + tolerations: + - operator: Exists serviceAccountName: csi-cinder-node-sa hostNetwork: true dnsPolicy: ClusterFirstWithHostNet diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 91a333dfb8c..273d4ddd212 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -359,8 +359,8 @@ snapshot_controller_supported_versions: snapshot_controller_image_repo: "{{ kube_image_repo }}/sig-storage/snapshot-controller" snapshot_controller_image_tag: "{{ snapshot_controller_supported_versions[kube_major_version] }}" -cinder_csi_plugin_version: "v1.22.0" -cinder_csi_plugin_image_repo: "{{ docker_image_repo }}/k8scloudprovider/cinder-csi-plugin" +cinder_csi_plugin_version: "v1.29.0" +cinder_csi_plugin_image_repo: "{{ kube_image_repo }}/provider-os/cinder-csi-plugin" cinder_csi_plugin_image_tag: "{{ cinder_csi_plugin_version }}" aws_ebs_csi_plugin_version: "v0.5.0" From 4317723d3c5cda77e3cfd7dbc6033fbb47006dbb Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Tue, 27 Feb 2024 16:24:34 +0800 Subject: [PATCH 102/103] cleanup-for-2.24.1 (#10962) --- .gitlab-ci.yml | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 45cece0f4dd..61cde2c9e41 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ stages: - deploy-special variables: - KUBESPRAY_VERSION: v2.24.0 + KUBESPRAY_VERSION: v2.24.1 FAILFASTCI_NAMESPACE: 'kargo-ci' GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray' ANSIBLE_FORCE_COLOR: "true" diff --git a/README.md b/README.md index 6e6e7610f59..308364fed44 100644 --- a/README.md +++ b/README.md @@ -75,11 +75,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou to access the inventory and SSH key in the container, like this: ```ShellSession -git checkout v2.24.0 -docker pull quay.io/kubespray/kubespray:v2.24.0 +git checkout v2.24.1 +docker pull quay.io/kubespray/kubespray:v2.24.1 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ - quay.io/kubespray/kubespray:v2.24.0 bash + quay.io/kubespray/kubespray:v2.24.1 bash # Inside the container you may now run the kubespray playbooks: ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml ``` From 7a9def547ec02b6821d1e9d9a5ce3ba41f4ff5e6 Mon Sep 17 00:00:00 2001 From: Clement Phu <34369901+cleman95@users.noreply.github.com> Date: Wed, 28 Feb 2024 03:10:19 +0100 Subject: [PATCH 103/103] Upgrade Helm to v3.14.2 (#10967) --- README.md | 2 +- .../defaults/main/checksums.yml | 16 ++++++++++++++++ .../defaults/main/download.yml | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 308364fed44..57ae5abd27b 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ Note: Upstart/SysV init based OS types are not supported. - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.6 - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4 - [argocd](https://argoproj.github.io/) v2.8.4 - - [helm](https://helm.sh/) v3.13.1 + - [helm](https://helm.sh/) v3.14.2 - [metallb](https://metallb.universe.tf/) v0.13.9 - [registry](https://github.com/distribution/distribution) v2.8.1 - Storage Plugin diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index 963ec2310e7..be30eb65834 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -643,6 +643,10 @@ krew_archive_checksums: v0.4.2: 0 helm_archive_checksums: arm: + v3.14.2: b70fb6fa2cdf0a5c782320c9d7e7b155fcaec260169218c98316bb3cf0d431d9 + v3.14.1: f50c00c262b74435530e677bcec07637aaeda1ed92ef809b49581a4e6182cbbe + v3.14.0: cf38dfdead7266ae56662743bda0c78655814f0adeca382d1b07a812bb1a599a + v3.13.3: 0170b15f3951be399e27e0cfdc21edb211d3b6b2698e078f993d9558d9446e3f v3.13.2: 06e8436bde78d53ddb5095ba146fe6c7001297c7dceb9ef6b68992c3ecfde770 v3.13.1: a9c188c1a79d2eb1721aece7c4e7cfcd56fa76d1e37bd7c9c05d3969bb0499b4 v3.13.0: bb2cdde0d12c55f65e88e7c398e67463e74bc236f68b7f307a73174b35628c2e @@ -656,6 +660,10 @@ helm_archive_checksums: v3.11.0: cddbef72886c82a123038883f32b04e739cc4bd7b9e5f869740d51e50a38be01 v3.10.3: dca718eb68c72c51fc7157c4c2ebc8ce7ac79b95fc9355c5427ded99e913ec4c arm64: + v3.14.2: c65d6a9557bb359abc2c0d26670de850b52327dc3976ad6f9e14c298ea3e1b61 + v3.14.1: f865b8ad4228fd0990bbc5b50615eb6cb9eb31c9a9ca7238401ed897bbbe9033 + v3.14.0: b29e61674731b15f6ad3d1a3118a99d3cc2ab25a911aad1b8ac8c72d5a9d2952 + v3.13.3: 44aaa094ae24d01e8c36e327e1837fd3377a0f9152626da088384c5bc6d94562 v3.13.2: f5654aaed63a0da72852776e1d3f851b2ea9529cb5696337202703c2e1ed2321 v3.13.1: 8c4a0777218b266a7b977394aaf0e9cef30ed2df6e742d683e523d75508d6efe v3.13.0: d12a0e73a7dbff7d89d13e0c6eb73f5095f72d70faea30531941d320678904d2 @@ -669,6 +677,10 @@ helm_archive_checksums: v3.11.0: 57d36ff801ce8c0201ce9917c5a2d3b4da33e5d4ea154320962c7d6fb13e1f2c v3.10.3: 260cda5ff2ed5d01dd0fd6e7e09bc80126e00d8bdc55f3269d05129e32f6f99d amd64: + v3.14.2: 0885a501d586c1e949e9b113bf3fb3290b0bbf74db9444a1d8c2723a143006a5 + v3.14.1: 75496ea824f92305ff7d28af37f4af57536bf5138399c824dff997b9d239dd42 + v3.14.0: f43e1c3387de24547506ab05d24e5309c0ce0b228c23bd8aa64e9ec4b8206651 + v3.13.3: bbb6e7c6201458b235f335280f35493950dcd856825ddcfd1d3b40ae757d5c7d v3.13.2: 55a8e6dce87a1e52c61e0ce7a89bf85b38725ba3e8deb51d4a08ade8a2c70b2d v3.13.1: 98c363564d00afd0cc3088e8f830f2a0eeb5f28755b3d8c48df89866374a1ed0 v3.13.0: 138676351483e61d12dfade70da6c03d471bbdcac84eaadeb5e1d06fa114a24f @@ -682,6 +694,10 @@ helm_archive_checksums: v3.11.0: 6c3440d829a56071a4386dd3ce6254eab113bc9b1fe924a6ee99f7ff869b9e0b v3.10.3: 950439759ece902157cf915b209b8d694e6f675eaab5099fb7894f30eeaee9a2 ppc64le: + v3.14.2: f3bc8582ff151e619cd285d9cdf9fef1c5733ee5522d8bed2ef680ef07f87223 + v3.14.1: 4d853ab8fe3462287c7272fbadd5f73531ecdd6fa0db37d31630e41ae1ae21de + v3.14.0: f1f9d3561724863edd4c06d89acb2e2fd8ae0f1b72058ceb891fa1c346ce5dbc + v3.13.3: 85afc540af42ebbb6e6a4fe270b04ce1fa27fa72845cd1d352feea0f55df1ffc v3.13.2: 11d96134cc4ec106c23cd8c163072e9aed6cd73e36a3da120e5876d426203f37 v3.13.1: f0d4ae95b4db25d03ced987e30d424564bd4727af6a4a0b7fca41f14203306fb v3.13.0: d9be0057c21ce5994885630340b4f2725a68510deca6e3c455030d83336e4797 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 273d4ddd212..300d297fc54 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -124,7 +124,7 @@ kube_ovn_version: "v1.11.5" kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" kube_router_version: "v2.0.0" multus_version: "v3.8" -helm_version: "v3.13.1" +helm_version: "v3.14.2" nerdctl_version: "1.7.1" krew_version: "v0.4.4" skopeo_version: "v1.13.2"