Merge branch 'kubernetes-sigs:master' into master

.gitlab-ci.yml

@@ -9,7 +9,7 @@ stages:
   - deploy-special
 
 variables:
-  KUBESPRAY_VERSION: v2.23.0
+  KUBESPRAY_VERSION: v2.23.1
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
   ANSIBLE_FORCE_COLOR: "true"

README.md

@@ -75,11 +75,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.23.0
-docker pull quay.io/kubespray/kubespray:v2.23.0
+git checkout v2.23.1
+docker pull quay.io/kubespray/kubespray:v2.23.1
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.23.0 bash
+  quay.io/kubespray/kubespray:v2.23.1 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -164,22 +164,22 @@ Note: Upstart/SysV init based OS types are not supported.
   - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.3
   - [etcd](https://github.com/etcd-io/etcd) v3.5.9
   - [docker](https://www.docker.com/) v20.10 (see note)
-  - [containerd](https://containerd.io/) v1.7.7
+  - [containerd](https://containerd.io/) v1.7.8
   - [cri-o](http://cri-o.io/) v1.27 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0
   - [calico](https://github.com/projectcalico/calico) v3.26.3
   - [cilium](https://github.com/cilium/cilium) v1.13.4
   - [flannel](https://github.com/flannel-io/flannel) v0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) v1.11.5
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.6.0
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v2.0.0
   - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) v3.8
   - [weave](https://github.com/weaveworks/weave) v2.8.1
   - [kube-vip](https://github.com/kube-vip/kube-vip) v0.5.12
 - Application
   - [cert-manager](https://github.com/jetstack/cert-manager) v1.12.5
   - [coredns](https://github.com/coredns/coredns) v1.10.1
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.3
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.4
   - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4
   - [argocd](https://argoproj.github.io/) v2.8.4
   - [helm](https://helm.sh/) v3.13.1

docs/ansible_collection.md

@@ -15,7 +15,7 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a
    collections:
    - name: https://github.com/kubernetes-sigs/kubespray
      type: git
-     version: v2.23.0
+     version: v2.23.1
    ```
 
 2. Install your collection

docs/vars.md

@@ -186,6 +186,8 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
 * *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin.
   [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overridden in inventory vars.
 
+* *crio_criu_support_enabled* - When set to `true`, enables the container checkpoint/restore in CRI-O. It's required to install [CRIU](https://criu.org/Installation) on the host when dumping/restoring checkpoints. And it's recommended to enable the feature gate `ContainerCheckpoint` so that the kubelet get a higher level API to simplify the operations (**Note**: It's still in experimental stage, just for container analytics so far). You can follow the [documentation](https://kubernetes.io/blog/2022/12/05/forensic-container-checkpointing-alpha/).
+
 * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a
   proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames
   that correspond to each node.

galaxy.yml

@@ -2,7 +2,7 @@
 namespace: kubernetes_sigs
 description: Deploy a production ready Kubernetes cluster
 name: kubespray
-version: 2.23.0
+version: 2.23.1
 readme: README.md
 authors:
   - luksi1

inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml

@@ -1,4 +1,10 @@
-# See roles/network_plugin/kube-router//defaults/main.yml
+# See roles/network_plugin/kube-router/defaults/main.yml
+
+# Kube router version
+# Default to v2
+# kube_router_version: "v2.0.0"
+# Uncomment to use v1 (Deprecated)
+# kube_router_version: "v1.6.0"
 
 # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
 # kube_router_run_router: true

roles/container-engine/containerd/handlers/main.yml

@@ -1,21 +1,17 @@
 ---
-- name: Restart containerd
-  command: /bin/true
-  notify:
-    - Containerd | restart containerd
-    - Containerd | wait for containerd
-
 - name: Containerd | restart containerd
   systemd:
     name: containerd
     state: restarted
     enabled: yes
     daemon-reload: yes
     masked: no
+  listen: Restart containerd
 
 - name: Containerd | wait for containerd
   command: "{{ containerd_bin_dir }}/ctr images ls -q"
   register: containerd_ready
   retries: 8
   delay: 4
   until: containerd_ready.rc == 0
+  listen: Restart containerd

roles/container-engine/cri-dockerd/handlers/main.yml

@@ -1,35 +1,31 @@
 ---
-- name: Restart and enable cri-dockerd
-  command: /bin/true
-  notify:
-    - Cri-dockerd | reload systemd
-    - Cri-dockerd | restart docker.service
-    - Cri-dockerd | reload cri-dockerd.socket
-    - Cri-dockerd | reload cri-dockerd.service
-    - Cri-dockerd | enable cri-dockerd service
-
 - name: Cri-dockerd | reload systemd
   systemd:
     name: cri-dockerd
     daemon_reload: true
     masked: no
+  listen: Restart and enable cri-dockerd
 
 - name: Cri-dockerd | restart docker.service
   service:
     name: docker.service
     state: restarted
+  listen: Restart and enable cri-dockerd
 
 - name: Cri-dockerd | reload cri-dockerd.socket
   service:
     name: cri-dockerd.socket
     state: restarted
+  listen: Restart and enable cri-dockerd
 
 - name: Cri-dockerd | reload cri-dockerd.service
   service:
     name: cri-dockerd.service
     state: restarted
+  listen: Restart and enable cri-dockerd
 
 - name: Cri-dockerd | enable cri-dockerd service
   service:
     name: cri-dockerd.service
     enabled: yes
+  listen: Restart and enable cri-dockerd

roles/container-engine/cri-o/defaults/main.yml

@@ -97,3 +97,6 @@ crio_man_files:
   8:
     - crio
     - crio-status
+
+# If set to true, it will enable the CRIU support in cri-o
+crio_criu_support_enabled: false

roles/container-engine/cri-o/handlers/main.yml

@@ -1,16 +1,12 @@
 ---
-- name: Restart crio
-  command: /bin/true
-  notify:
-    - CRI-O | reload systemd
-    - CRI-O | reload crio
-
 - name: CRI-O | reload systemd
   systemd:
     daemon_reload: true
+  listen: Restart crio
 
 - name: CRI-O | reload crio
   service:
     name: crio
     state: restarted
     enabled: yes
+  listen: Restart crio

roles/container-engine/cri-o/tasks/cleanup.yaml

@@ -118,3 +118,8 @@
     - cri-o
     - cri-o-runc
     - oci-systemd-hook
+
+- name: CRI-O | Remove CRI-O package configuration files
+  file:
+    name: /etc/crio/crio.conf.d/01-crio-runc.conf
+    state: absent

roles/container-engine/cri-o/tasks/reset.yml

@@ -17,7 +17,7 @@
 - name: CRI-O | Remove cri-o apt repo
   apt_repository:
     repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
-    state: present
+    state: absent
     filename: devel-kubic-libcontainers-stable-cri-o
   when: crio_kubic_debian_repo_name is defined
   tags:

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -273,6 +273,11 @@ pinns_path = ""
 pinns_path = "{{ bin_dir }}/pinns"
 {% endif %}
 
+{% if crio_criu_support_enabled %}
+# Enable CRIU integration, requires that the criu binary is available in $PATH.
+enable_criu_support = true
+{% endif %}
+
 # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
 # The runtime to use is picked based on the runtime_handler provided by the CRI.
 # If no runtime_handler is provided, the runtime will be picked based on the level

roles/container-engine/docker/handlers/main.yml

@@ -1,32 +1,30 @@
 ---
-- name: Restart docker
-  command: /bin/true
-  notify:
-    - Docker | reload systemd
-    - Docker | reload docker.socket
-    - Docker | reload docker
-    - Docker | wait for docker
-
 - name: Docker | reload systemd
   systemd:
     name: docker
     daemon_reload: true
     masked: no
+  listen: Restart docker
 
 - name: Docker | reload docker.socket
   service:
     name: docker.socket
     state: restarted
   when: ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos
+  listen: Restart docker
+
 
 - name: Docker | reload docker
   service:
     name: docker
     state: restarted
+  listen: Restart docker
+
 
 - name: Docker | wait for docker
   command: "{{ docker_bin_dir }}/docker images"
   register: docker_ready
   retries: 20
   delay: 1
   until: docker_ready.rc == 0
+  listen: Restart docker

roles/download/defaults/main/checksums.yml

@@ -1045,6 +1045,7 @@ nerdctl_archive_checksums:
     1.0.0: 2fb02e629a4be16b194bbfc64819132a72ede1f52596bd8e1ec2beaf7c28c117
 containerd_archive_checksums:
   arm:
+    1.7.8: 0
     1.7.7: 0
     1.7.6: 0
     1.7.5: 0
@@ -1084,6 +1085,7 @@ containerd_archive_checksums:
     1.5.15: 0
     1.5.14: 0
   arm64:
+    1.7.8: 3fc551e8f51150804d80cc1958a271bd2252b6334f0355244d0faa5da7fa55d1
     1.7.7: 0a104f487193665d2681fcb5ed83f2baa5f97849fe2661188da835c9d4eaf9e3
     1.7.6: d844a1c8b993e7e9647f73b9814567004dce1287c0529ce55c50519490eafcce
     1.7.5: 98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9
@@ -1123,6 +1125,7 @@ containerd_archive_checksums:
     1.5.15: 0
     1.5.14: 0
   amd64:
+    1.7.8: 5f1d017a5a7359514d6187d6656e88fb2a592d107e6298db7963dbddb9a111d9
     1.7.7: 371de359d6102c51f6ee2361d08297948d134ce7379e01cb965ceeffa4365fba
     1.7.6: 58408cfa025003e671b0af72183b963363d519543d0d0ba186037e9c57489ffe
     1.7.5: 33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25
@@ -1162,6 +1165,7 @@ containerd_archive_checksums:
     1.5.15: 0d09043be08dcf6bf136aa78bfd719e836cf9f9679afa4db0b6e4d478e396528
     1.5.14: 8513ead11aca164b7e70bcea0429b4e51dad836b6383b806322e128821aaebbd
   ppc64le:
+    1.7.8: 2b563df9e1bddc96a99a023963c99b5faf3066d3fcbc23ff44ba24229e939444
     1.7.7: 0335e7447ed84757489337686a709e95ffa379a8780f238725abb10facaeaa7f
     1.7.6: 956fadb01b35c3214f2b6f82abc0dda3e1b754cb223cd24e818334b08cb09fb2
     1.7.5: 2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc

roles/download/defaults/main/main.yml

@@ -79,7 +79,7 @@ runc_version: v1.1.9
 kata_containers_version: 3.1.3
 youki_version: 0.1.0
 gvisor_version: 20230807
-containerd_version: 1.7.7
+containerd_version: 1.7.8
 cri_dockerd_version: 0.3.4
 
 # this is relevant when container_manager == 'docker'
@@ -122,7 +122,7 @@ cilium_enable_hubble: false
 
 kube_ovn_version: "v1.11.5"
 kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}"
-kube_router_version: "v1.6.0"
+kube_router_version: "v2.0.0"
 multus_version: "v3.8"
 helm_version: "v3.13.1"
 nerdctl_version: "1.6.0"
@@ -306,7 +306,7 @@ rbd_provisioner_image_tag: "{{ rbd_provisioner_version }}"
 local_path_provisioner_version: "v0.0.24"
 local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
 local_path_provisioner_image_tag: "{{ local_path_provisioner_version }}"
-ingress_nginx_version: "v1.9.3"
+ingress_nginx_version: "v1.9.4"
 ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller"
 ingress_nginx_controller_image_tag: "{{ ingress_nginx_version }}"
 ingress_nginx_kube_webhook_certgen_image_repo: "{{ kube_image_repo }}/ingress-nginx/kube-webhook-certgen"

roles/etcd/handlers/backup.yml

@@ -1,22 +1,14 @@
 ---
-- name: Backup etcd data
-  command: /bin/true
-  notify:
-    - Refresh Time Fact
-    - Set Backup Directory
-    - Create Backup Directory
-    - Stat etcd v2 data directory
-    - Backup etcd v2 data
-    - Backup etcd v3 data
-  when: etcd_cluster_is_healthy.rc == 0
-
 - name: Refresh Time Fact
   setup:
     filter: ansible_date_time
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0
 
 - name: Set Backup Directory
   set_fact:
     etcd_backup_directory: "{{ etcd_backup_prefix }}/etcd-{{ ansible_date_time.date }}_{{ ansible_date_time.time }}"
+  listen: Restart etcd
 
 - name: Create Backup Directory
   file:
@@ -25,6 +17,8 @@
     owner: root
     group: root
     mode: 0600
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0
 
 - name: Stat etcd v2 data directory
   stat:
@@ -33,9 +27,13 @@
     get_checksum: no
     get_mime: no
   register: etcd_data_dir_member
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0
 
 - name: Backup etcd v2 data
-  when: etcd_data_dir_member.stat.exists
+  when:
+    - etcd_data_dir_member.stat.exists
+    - etcd_cluster_is_healthy.rc == 0
   command: >-
     {{ bin_dir }}/etcdctl backup
       --data-dir {{ etcd_data_dir }}
@@ -46,6 +44,7 @@
   register: backup_v2_command
   until: backup_v2_command.rc == 0
   delay: "{{ retry_stagger | random + 3 }}"
+  listen: Restart etcd
 
 - name: Backup etcd v3 data
   command: >-
@@ -61,3 +60,5 @@
   register: etcd_backup_v3_command
   until: etcd_backup_v3_command.rc == 0
   delay: "{{ retry_stagger | random + 3 }}"
+  listen: Restart etcd
+  when: etcd_cluster_is_healthy.rc == 0

roles/etcd/handlers/backup_cleanup.yml

@@ -1,10 +1,4 @@
 ---
-- name: Cleanup etcd backups
-  command: /bin/true
-  notify:
-    - Find old etcd backups
-    - Remove old etcd backups
-
 - name: Find old etcd backups
   ansible.builtin.find:
     file_type: directory
@@ -13,10 +7,12 @@
     patterns: "etcd-*"
   register: _etcd_backups
   when: etcd_backup_retention_count >= 0
+  listen: Restart etcd
 
 - name: Remove old etcd backups
   ansible.builtin.file:
     state: absent
     path: "{{ item }}"
   loop: "{{ (_etcd_backups.files | sort(attribute='ctime', reverse=True))[etcd_backup_retention_count:] | map(attribute='path') }}"
   when: etcd_backup_retention_count >= 0
+  listen: Restart etcd